├── .gitignore ├── files ├── td.repo └── td-agent.conf ├── tasks ├── set-role-variables.yml ├── use-yum.yml ├── use-apt.yml └── main.yml ├── defaults └── main.yml ├── handlers └── main.yml ├── meta └── main.yml ├── test ├── Dockerfile-centos6 ├── Dockerfile-centos7 ├── Dockerfile-ubuntu12.04 ├── Dockerfile-ubuntu14.04 ├── Dockerfile-ubuntu16.04 ├── Dockerfile-debian7 ├── Dockerfile-debian8 └── prometheus.conf.j2 ├── test.yml ├── LICENSE ├── circle.yml ├── .travis.yml ├── Vagrantfile └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | -------------------------------------------------------------------------------- /files/td.repo: -------------------------------------------------------------------------------- 1 | [treasuredata] 2 | name=TreasureData 3 | baseurl=http://packages.treasuredata.com/2/redhat/$releasever/$basearch 4 | gpgcheck=1 5 | gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent -------------------------------------------------------------------------------- /tasks/set-role-variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: tasks/set-role-variables.yml 3 | # set necessary role variables. 4 | # 5 | 6 | - name: set tdagent_use_service = True, if not defined 7 | set_fact: 8 | tdagent_use_service: True 9 | when: tdagent_use_service is not defined 10 | -------------------------------------------------------------------------------- /defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # 4 | # variables needed to be defined in user's playbook 5 | # 6 | 7 | 8 | 9 | 10 | # 11 | # user-configurable defaults 12 | # 13 | 14 | 15 | #tdagent_version 16 | 17 | 18 | #---- the following vars are handled in tasks/set-role-variables.yml ------ 19 | 20 | #tdagent_use_service: True # use "service" command to start/restart fluentd? 21 | -------------------------------------------------------------------------------- /handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: handlers/main.yml 3 | # 4 | 5 | - name: reload td-agent 6 | service: name=td-agent state=reloaded 7 | when: tdagent_use_service|bool 8 | 9 | - name: restart td-agent 10 | service: name=td-agent state=restarted 11 | when: tdagent_use_service|bool 12 | 13 | - name: stop td-agent 14 | service: name=td-agent state=stopped 15 | when: tdagent_use_service|bool 16 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # meta/main.yml 3 | 4 | galaxy_info: 5 | author: williamyeh 6 | description: Install Fluentd (td-agent version) 7 | license: MIT 8 | min_ansible_version: 1.8 9 | 10 | platforms: 11 | 12 | - name: Ubuntu 13 | versions: 14 | - precise 15 | - trusty 16 | - xenial 17 | 18 | - name: Debian 19 | versions: 20 | - wheezy 21 | - jessie 22 | 23 | - name: EL 24 | versions: 25 | - 6 26 | - 7 27 | 28 | galaxy_tags: 29 | - system 30 | - logging 31 | - fluentd 32 | - monitoring 33 | 34 | dependencies: [] 35 | -------------------------------------------------------------------------------- /files/td-agent.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Simple default td-agent.conf 3 | # For more details, see http://docs.fluentd.org/articles/config-file 4 | # 5 | 6 | 7 | # built-in TCP input to receive event logs 8 | # @see http://docs.fluentd.org/articles/in_forward 9 | 10 | @type forward 11 | #port 24224 12 | #bind 0.0.0.0 13 | 14 | 15 | 16 | # HTTP input via POST 17 | # @see http://docs.fluentd.org/articles/in_http 18 | 19 | @type http 20 | #port 9880 21 | #bind 0.0.0.0 22 | #keepalive_timeout 10s 23 | 24 | 25 | 26 | # Monitoring Agent: retrieve internal metrics in JSON via HTTP 27 | # @see http://docs.fluentd.org/articles/monitoring 28 | 29 | @type monitor_agent 30 | #bind 0.0.0.0 31 | #port 24220 32 | 33 | 34 | 35 | @include /etc/td-agent/conf.d/*.conf -------------------------------------------------------------------------------- /test/Dockerfile-centos6: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:centos6-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | ENV PLAYBOOK test.yml 22 | RUN ansible-playbook-wrapper -vvv 23 | 24 | # port for monitoring agent 25 | EXPOSE 24220 26 | 27 | 28 | # 29 | # test phase 30 | # 31 | 32 | RUN echo "===> Installing curl for testing purpose..." && \ 33 | yum -y install curl 34 | 35 | 36 | VOLUME ["/data"] 37 | ENV RESULT /data/result-centos6 38 | 39 | CMD service td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT 40 | -------------------------------------------------------------------------------- /test.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: True 3 | tasks: 4 | - include: 'tasks/main.yml' 5 | handlers: 6 | - include: 'handlers/main.yml' 7 | vars_files: 8 | - 'defaults/main.yml' 9 | 10 | vars: 11 | #tdagent_version: 2.3.4 12 | 13 | #tdagent_plugins: 14 | # - fluent-plugin-multiprocess 15 | # - fluent-plugin-anonymizer 16 | # - fluent-plugin-forest 17 | # - fluent-plugin-record-reformer 18 | # #- fluent-plugin-geoip 19 | # #- fluent-plugin-elasticsearch 20 | 21 | tdagent_plugins_versions: 22 | prometheus: 23 | name: fluent-plugin-prometheus 24 | version: 0.1.2 25 | flowcounter: 26 | name: fluent-plugin-flowcounter 27 | version: 0.4.1 28 | 29 | 30 | tdagent_conf_copy: "files/td-agent.conf" 31 | 32 | tdagent_conf_others: 33 | prometheus_metrics: 34 | src: test/prometheus.conf.j2 35 | dest: prometheus.conf 36 | -------------------------------------------------------------------------------- /test/Dockerfile-centos7: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:centos7-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | ENV PLAYBOOK test.yml 22 | RUN ansible-playbook-wrapper -vvv --extra-vars "tdagent_use_service=False" 23 | 24 | # port for monitoring agent 25 | EXPOSE 24220 26 | 27 | 28 | # 29 | # test phase 30 | # 31 | 32 | RUN echo "===> Installing curl for testing purpose..." && \ 33 | yum -y install curl 34 | 35 | 36 | VOLUME ["/data"] 37 | ENV RESULT /data/result-centos7 38 | 39 | CMD /etc/init.d/td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT 40 | -------------------------------------------------------------------------------- /test/Dockerfile-ubuntu12.04: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:ubuntu12.04-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | ENV PLAYBOOK test.yml 22 | RUN ansible-playbook-wrapper 23 | 24 | # port for monitoring agent 25 | EXPOSE 24220 26 | 27 | 28 | # 29 | # test phase 30 | # 31 | 32 | RUN echo "===> Installing curl for testing purpose..." && \ 33 | DEBIAN_FRONTEND=noninteractive \ 34 | apt-get install -y -f curl 35 | 36 | 37 | VOLUME ["/data"] 38 | ENV RESULT /data/result-ubuntu12.04 39 | 40 | CMD service td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT -------------------------------------------------------------------------------- /test/Dockerfile-ubuntu14.04: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:ubuntu14.04-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | ENV PLAYBOOK test.yml 22 | RUN ansible-playbook-wrapper 23 | 24 | # port for monitoring agent 25 | EXPOSE 24220 26 | 27 | 28 | # 29 | # test phase 30 | # 31 | 32 | RUN echo "===> Installing curl for testing purpose..." && \ 33 | DEBIAN_FRONTEND=noninteractive \ 34 | apt-get install -y -f curl 35 | 36 | 37 | VOLUME ["/data"] 38 | ENV RESULT /data/result-ubuntu14.04 39 | 40 | CMD service td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT -------------------------------------------------------------------------------- /test/Dockerfile-ubuntu16.04: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:ubuntu16.04-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | ENV PLAYBOOK test.yml 22 | RUN ansible-playbook-wrapper 23 | 24 | # port for monitoring agent 25 | EXPOSE 24220 26 | 27 | 28 | # 29 | # test phase 30 | # 31 | 32 | RUN echo "===> Installing curl for testing purpose..." && \ 33 | DEBIAN_FRONTEND=noninteractive \ 34 | apt-get install -y -f curl 35 | 36 | 37 | VOLUME ["/data"] 38 | ENV RESULT /data/result-ubuntu16.04 39 | 40 | CMD service td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT -------------------------------------------------------------------------------- /test/Dockerfile-debian7: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:debian7-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | # install commands required in http://packages.treasuredata.com/2/debian/wheezy/pool/contrib/t/td-agent/ 22 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y adduser 23 | 24 | ENV PLAYBOOK test.yml 25 | RUN ansible-playbook-wrapper -vvv 26 | 27 | # port for monitoring agent 28 | EXPOSE 24220 29 | 30 | 31 | # 32 | # test phase 33 | # 34 | 35 | RUN echo "===> Installing curl for testing purpose..." && \ 36 | DEBIAN_FRONTEND=noninteractive \ 37 | apt-get install -y -f curl 38 | 39 | 40 | VOLUME ["/data"] 41 | ENV RESULT /data/result-debian7 42 | 43 | CMD service td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT -------------------------------------------------------------------------------- /test/Dockerfile-debian8: -------------------------------------------------------------------------------- 1 | # Dockerfile for building image that contains software stack provisioned by Ansible. 2 | # 3 | # USAGE: 4 | # $ docker build -t fluentd . 5 | # $ docker run -p 24220:24220 fluentd 6 | # 7 | # Version 1.0 8 | # 9 | 10 | 11 | # pull base image 12 | FROM williamyeh/ansible:debian8-onbuild 13 | 14 | MAINTAINER William Yeh 15 | 16 | 17 | # 18 | # build phase 19 | # 20 | 21 | # install commands required in http://packages.treasuredata.com/2/debian/jessie/pool/contrib/t/td-agent/ 22 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y adduser 23 | 24 | ENV PLAYBOOK test.yml 25 | RUN ansible-playbook-wrapper -vvv 26 | 27 | # port for monitoring agent 28 | EXPOSE 24220 29 | 30 | 31 | # 32 | # test phase 33 | # 34 | 35 | RUN echo "===> Installing curl for testing purpose..." && \ 36 | DEBIAN_FRONTEND=noninteractive \ 37 | apt-get install -y -f curl 38 | 39 | 40 | VOLUME ["/data"] 41 | ENV RESULT /data/result-debian8 42 | 43 | CMD service td-agent start && sleep 10 && curl --retry 5 --retry-max-time 120 http://localhost:24220/api/plugins.json > $RESULT -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 William Yeh 4 | 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | services: 3 | - docker 4 | 5 | dependencies: 6 | override: 7 | - docker info 8 | - docker version 9 | 10 | - docker build -f test/Dockerfile-ubuntu16.04 -t fluentd_xenial . 11 | - docker build -f test/Dockerfile-ubuntu14.04 -t fluentd_trusty . 12 | - docker build -f test/Dockerfile-ubuntu12.04 -t fluentd_precise . 13 | - docker build -f test/Dockerfile-debian8 -t fluentd_jessie . 14 | - docker build -f test/Dockerfile-debian7 -t fluentd_wheezy . 15 | - docker build -f test/Dockerfile-centos7 -t fluentd_centos7 . 16 | - docker build -f test/Dockerfile-centos6 -t fluentd_centos6 . 17 | 18 | test: 19 | override: 20 | - docker run -v $(pwd):/data fluentd_xenial 21 | - docker run -v $(pwd):/data fluentd_trusty 22 | - docker run -v $(pwd):/data fluentd_precise 23 | - docker run -v $(pwd):/data fluentd_jessie 24 | - docker run -v $(pwd):/data fluentd_wheezy 25 | - docker run -v $(pwd):/data fluentd_centos7 26 | - docker run -v $(pwd):/data fluentd_centos6 27 | 28 | - echo "==> Validating the test results..." 29 | - grep 'monitor_agent' result-ubuntu16.04 30 | - grep 'monitor_agent' result-ubuntu14.04 31 | - grep 'monitor_agent' result-ubuntu12.04 32 | - grep 'monitor_agent' result-debian8 33 | - grep 'monitor_agent' result-debian7 34 | - grep 'monitor_agent' result-centos7 35 | - grep 'monitor_agent' result-centos6 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | services: 3 | - docker 4 | 5 | before_install: 6 | - docker info 7 | - docker version 8 | 9 | - docker build -f test/Dockerfile-ubuntu16.04 -t fluentd_xenial . 10 | - docker build -f test/Dockerfile-ubuntu14.04 -t fluentd_trusty . 11 | - docker build -f test/Dockerfile-ubuntu12.04 -t fluentd_precise . 12 | - docker build -f test/Dockerfile-debian8 -t fluentd_jessie . 13 | - docker build -f test/Dockerfile-debian7 -t fluentd_wheezy . 14 | - docker build -f test/Dockerfile-centos7 -t fluentd_centos7 . 15 | - docker build -f test/Dockerfile-centos6 -t fluentd_centos6 . 16 | 17 | script: 18 | - docker run -v $(pwd):/data fluentd_xenial 19 | - docker run -v $(pwd):/data fluentd_trusty 20 | - docker run -v $(pwd):/data fluentd_precise 21 | - docker run -v $(pwd):/data fluentd_jessie 22 | - docker run -v $(pwd):/data fluentd_wheezy 23 | - docker run -v $(pwd):/data fluentd_centos7 24 | - docker run -v $(pwd):/data fluentd_centos6 25 | 26 | - echo "==> Validating the test results..." 27 | - grep 'monitor_agent' result-ubuntu16.04 28 | - grep 'monitor_agent' result-ubuntu14.04 29 | - grep 'monitor_agent' result-ubuntu12.04 30 | - grep 'monitor_agent' result-debian8 31 | - grep 'monitor_agent' result-debian7 32 | - grep 'monitor_agent' result-centos7 33 | - grep 'monitor_agent' result-centos6 34 | 35 | notifications: 36 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /tasks/use-yum.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: tasks/use-yum.yml 3 | # 4 | # Configure td-agent from YUM repository. 5 | # 6 | # @see http://docs.fluentd.org/articles/install-by-rpm 7 | # @see http://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh 8 | # 9 | 10 | 11 | - name: install libselinux-python binary for Ansible to work 12 | yum: name=libselinux-python state=present 13 | 14 | 15 | 16 | - name: add GPG key from Treasure Data, Inc 17 | command: rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent 18 | # rpm_key: key=https://packages.treasuredata.com/GPG-KEY-td-agent validate_certs=no state=present 19 | 20 | - name: add official repository 21 | copy: src=../files/td.repo dest=/etc/yum.repos.d/td.repo 22 | 23 | - name: install td-agent 24 | yum: name=td-agent update_cache=yes state=present 25 | when: tdagent_version is not defined 26 | 27 | - name: install td-agent 28 | yum: name="td-agent-{{ tdagent_version }}*" update_cache=yes state=present 29 | when: tdagent_version is defined 30 | 31 | 32 | - name: install gcc and libcurlXXX for compiling plugins 33 | yum: name={{ item }} state=present 34 | with_items: 35 | - gcc 36 | - libcurl 37 | - libcurl-devel 38 | when: tdagent_plugins is defined or tdagent_plugins_versions is defined 39 | 40 | 41 | - name: set INIT status and start 42 | service: name=td-agent state=started enabled=yes 43 | when: tdagent_use_service|bool 44 | 45 | - name: set INIT status (SysV style) 46 | command: /sbin/chkconfig td-agent on 47 | when: not tdagent_use_service|bool 48 | 49 | #- name: set INIT status (SysV style) 50 | # action: shell chkconfig --add td-agent 51 | # action: shell chkconfig --level 345 td-agent on 52 | -------------------------------------------------------------------------------- /tasks/use-apt.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: tasks/use-apt.yml 3 | # 4 | # Configure td-agent from APT repository. 5 | # 6 | # @see http://docs.fluentd.org/articles/install-by-deb 7 | # @see http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh 8 | # 9 | 10 | - name: install apt-related binaries for Ansible to work 11 | apt: name={{ item }} state=present update_cache=yes 12 | with_items: 13 | - python-software-properties 14 | - apt-transport-https 15 | 16 | - name: add APT signing key for td-agent 17 | apt_key: url=https://packages.treasuredata.com/GPG-KEY-td-agent state=present 18 | 19 | - name: add td-agent repository 20 | apt_repository: repo='deb https://packages.treasuredata.com/2/{{ ansible_distribution|lower }}/{{ ansible_distribution_release|lower }}/ {{ ansible_distribution_release|lower }} contrib' state=present 21 | 22 | 23 | - name: install td-agent 24 | apt: name=td-agent update_cache=yes state=present force=yes 25 | when: tdagent_version is not defined 26 | 27 | - name: install td-agent 28 | apt: name="td-agent={{ tdagent_version }}*" update_cache=yes state=present force=yes 29 | when: tdagent_version is defined 30 | 31 | 32 | 33 | - name: install libcurl and make for compiling plugins 34 | apt: name={{ item }} state=present update_cache=yes 35 | with_items: 36 | - libcurl4-gnutls-dev 37 | - build-essential 38 | when: tdagent_plugins is defined or tdagent_plugins_versions is defined 39 | 40 | 41 | 42 | - name: set INIT status and start 43 | service: name=td-agent state=started enabled=yes 44 | when: tdagent_use_service|bool 45 | 46 | - name: set INIT status (SysV style) 47 | shell: update-rc.d td-agent defaults 48 | when: not tdagent_use_service|bool 49 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure(2) do |config| 2 | 3 | # main & default: normal OS series... 4 | config.vm.define "main", primary: true do |node| 5 | node.vm.box = "ubuntu/trusty64" 6 | #node.vm.box = "ubuntu/precise64" 7 | #node.vm.box = "debian/jessie64" 8 | #node.vm.box = "debian/wheezy64" 9 | #node.vm.box = "bento/centos-7.2" 10 | #node.vm.box = "bento/centos-6.7" 11 | 12 | node.vm.network "forwarded_port", guest: 24224, host: 24224 13 | node.vm.network "forwarded_port", guest: 9880, host: 9880 14 | #node.vm.network "forwarded_port", guest: 24220, host: 24220 15 | node.vm.network "forwarded_port", guest: 24231, host: 24231 16 | 17 | node.vm.provision "ansible" do |ansible| 18 | ansible.playbook = "test.yml" 19 | #ansible.sudo = true 20 | ansible.verbose = "vvv" 21 | end 22 | end 23 | 24 | 25 | # docker: for auto build & testing (e.g., Travis CI) 26 | config.vm.define "docker" do |node| 27 | node.vm.box = "williamyeh/ubuntu-trusty64-docker" 28 | 29 | node.vm.provision "shell", inline: <<-SHELL 30 | cd /vagrant 31 | docker build -f test/Dockerfile-ubuntu14.04 -t fluentd_trusty . 32 | docker build -f test/Dockerfile-ubuntu12.04 -t fluentd_precise . 33 | docker build -f test/Dockerfile-debian8 -t fluentd_jessie . 34 | docker build -f test/Dockerfile-debian7 -t fluentd_wheezy . 35 | docker build -f test/Dockerfile-centos7 -t fluentd_centos7 . 36 | docker build -f test/Dockerfile-centos6 -t fluentd_centos6 . 37 | SHELL 38 | end 39 | 40 | end 41 | 42 | -------------------------------------------------------------------------------- /test/prometheus.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # More elaborated td-agent.conf with Prometheus. 3 | # For more details, see http://docs.fluentd.org/articles/config-file 4 | # 5 | 6 | 7 | #---------------------------------------# 8 | # Source 9 | # 10 | 11 | # Fluentd Exporter for Prometheus 12 | # @see https://github.com/kazegusuri/fluent-plugin-prometheus 13 | # 14 | # endpoint definition 15 | 16 | @type prometheus 17 | #bind 0.0.0.0 18 | #port 24231 19 | #metrics_path /metrics 20 | 21 | # input plugin that collects metrics from MonitorAgent 22 | 23 | @type prometheus_monitor 24 | 25 | host ${hostname} 26 | 27 | 28 | 29 | 30 | 31 | #---------------------------------------# 32 | # Filter 33 | # 34 | 35 | 36 | 37 | #---------------------------------------# 38 | # Match 39 | # 40 | 41 | 42 | @type copy 43 | 44 | @type stdout 45 | 46 | 47 | @type prometheus 48 | 49 | name fluentd_count_per_minute 50 | type gauge 51 | desc The average traffic items of fluentd 52 | key count 53 | 54 | 55 | name fluentd_bytes_per_minute 56 | type gauge 57 | desc The average traffic bytes of fluentd 58 | key bytes 59 | 60 | 61 | 62 | 63 | 64 | 65 | @type copy 66 | # 67 | # @type stdout 68 | # 69 | 70 | @type flowcounter 71 | count_keys * 72 | unit minute 73 | aggregate all 74 | tag fluentd.traffic.minute 75 | 76 | 77 | -------------------------------------------------------------------------------- /tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: tasks/main.yml 3 | # Top-level installer for Fluentd. 4 | # 5 | # @see http://docs.treasuredata.com/articles/td-agent 6 | # 7 | 8 | - name: set role variables, if necessary 9 | include: set-role-variables.yml 10 | 11 | 12 | - name: delegate to APT system for installation 13 | include: use-apt.yml 14 | when: ansible_pkg_mgr == "apt" 15 | 16 | - name: delegate to YUM system for installation 17 | include: use-yum.yml 18 | when: ansible_pkg_mgr == "yum" 19 | 20 | 21 | 22 | 23 | - name: mkdir for conf.d 24 | file: 25 | path: /etc/td-agent/conf.d 26 | state: directory 27 | owner: td-agent 28 | group: td-agent 29 | 30 | - name: rename default td-agent.conf 31 | command: creates="/etc/td-agent/td-agent.conf.bak" mv /etc/td-agent/td-agent.conf /etc/td-agent/td-agent.conf.bak 32 | 33 | 34 | 35 | - name: copy new td-agent.conf from role's default 36 | copy: 37 | src: ../files/td-agent.conf 38 | dest: /etc/td-agent/td-agent.conf 39 | owner: td-agent 40 | group: td-agent 41 | when: tdagent_conf_copy is not defined and tdagent_conf_template is not defined 42 | notify: 43 | - reload td-agent 44 | 45 | - name: copy new "td-agent.conf" verbatim from main playbook's 46 | copy: 47 | src: "{{ playbook_dir }}/{{ tdagent_conf_copy }}" 48 | dest: /etc/td-agent/td-agent.conf 49 | owner: td-agent 50 | group: td-agent 51 | when: tdagent_conf_copy is defined 52 | notify: 53 | - reload td-agent 54 | 55 | - name: copy (from template) new "td-agent.conf" from main playbook's 56 | template: 57 | src: "{{ playbook_dir }}/{{ tdagent_conf_template }}" 58 | dest: /etc/td-agent/td-agent.conf 59 | owner: td-agent 60 | group: td-agent 61 | when: tdagent_conf_template is defined 62 | notify: 63 | - reload td-agent 64 | 65 | - name: copy other config files from playbook's, if any 66 | template: 67 | src: "{{ playbook_dir }}/{{ item.value.src }}" 68 | dest: "/etc/td-agent/conf.d/{{ item.value.dest }}" 69 | owner: td-agent 70 | group: td-agent 71 | with_dict: '{{ tdagent_conf_others | default({}) }}' 72 | notify: 73 | - reload td-agent 74 | 75 | 76 | 77 | 78 | # GEM installed on: 79 | # /opt/td-agent/embedded/lib/ruby/gems/2.1.0/gems/ 80 | - name: install plugins, if any 81 | gem: 82 | name: "{{ item }}" 83 | executable: /opt/td-agent/embedded/bin/fluent-gem 84 | state: latest 85 | user_install: no 86 | with_items: "{{ tdagent_plugins | default([]) }}" 87 | notify: 88 | - restart td-agent 89 | 90 | 91 | - name: install plugins with specified versions, if any 92 | gem: 93 | name: "{{ item.value.name }}" 94 | executable: /opt/td-agent/embedded/bin/fluent-gem 95 | state: present 96 | version: "{{ item.value.version }}" 97 | user_install: no 98 | with_dict: "{{ tdagent_plugins_versions | default({}) }}" 99 | notify: 100 | - restart td-agent 101 | 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | williamyeh.fluentd for Ansible Galaxy 3 | ============ 4 | 5 | 6 | [![Circle CI](https://circleci.com/gh/William-Yeh/ansible-fluentd.svg?style=shield)](https://circleci.com/gh/William-Yeh/ansible-fluentd) [![Build Status](https://travis-ci.org/William-Yeh/ansible-fluentd.svg?branch=master)](https://travis-ci.org/William-Yeh/ansible-fluentd) 7 | 8 | 9 | ## Summary 10 | 11 | Role name in Ansible Galaxy: **[williamyeh.fluentd](https://galaxy.ansible.com/williamyeh/fluentd/)** 12 | 13 | This Ansible role has the following features for [Fluentd](http://www.fluentd.org/): 14 | 15 | - Install [td-agent](https://docs.treasuredata.com/articles/td-agent): the stable Fluentd distribution package maintained by [Treasure Data, Inc](https://www.treasuredata.com/). 16 | - Install several plugins. 17 | - Bare bone configuration (*real* configuration should be left to user's template files; see **Usage** section below). 18 | 19 | 20 | 21 | ## Role Variables 22 | 23 | ### Mandatory variables 24 | 25 | None. 26 | 27 | 28 | ### Optional variables 29 | 30 | User-configurable defaults: 31 | 32 | ```yaml 33 | # td-agent version; e.g., 2.3.4 34 | # Will install the default (usually the latest stable) version, if not specified. 35 | tdagent_version 36 | 37 | 38 | # an array of plugins (with latest versions) to be installed 39 | tdagent_plugins 40 | 41 | # an dict of plugins (with specified versions) to be installed 42 | # dict fields: 43 | # - key: memo for this plugin 44 | # - value: 45 | # - name: plugin name 46 | # - version: plugin version 47 | tdagent_plugins_versions 48 | ``` 49 | 50 | User-installable configuration files - main configuration: 51 | 52 | ```yaml 53 | # conf file (usually td-agent.conf) to be installed, 54 | # relative to `playbook_dir`; 55 | # the file will be copied verbatim 56 | tdagent_conf_copy 57 | 58 | # conf file (usually td-agent.conf.j2) to be installed, 59 | # relative to `playbook_dir`; 60 | # the file will be copied through Ansible's template system 61 | tdagent_conf_template 62 | ``` 63 | 64 | User-installable configuration files - other configurations: 65 | 66 | ```yaml 67 | # other conf templates to be installed to "/etc/td-agent/conf.d"; 68 | # dict fields: 69 | # - key: memo for this conf 70 | # - value: 71 | # - src: template file relative to `playbook_dir` 72 | # - dest: target file relative to `/etc/td-agent/conf.d/` 73 | tdagent_conf_others 74 | ``` 75 | 76 | 77 | 78 | ## Handlers 79 | 80 | - `reload td-agent` 81 | 82 | - `restart td-agent` 83 | 84 | - `stop td-agent` 85 | 86 | 87 | 88 | 89 | ## Usage 90 | 91 | 92 | ### Step 1: add role 93 | 94 | Add role name `williamyeh.fluentd` to your playbook file. 95 | 96 | 97 | ### Step 2: add variables, if any 98 | 99 | Set vars in your playbook file. 100 | 101 | Simple example: 102 | 103 | ```yaml 104 | --- 105 | # file: simple-playbook.yml 106 | 107 | - hosts: all 108 | 109 | roles: 110 | - williamyeh.fluentd 111 | 112 | vars: 113 | tdagent_plugins: 114 | - fluent-plugin-multiprocess 115 | - fluent-plugin-forest 116 | - fluent-plugin-elasticsearch 117 | 118 | tdagent_plugins_versions: 119 | prometheus: 120 | name: fluent-plugin-prometheus 121 | version: 0.1.2 122 | flowcounter: 123 | name: fluent-plugin-flowcounter 124 | version: 0.4.1 125 | 126 | ``` 127 | 128 | 129 | ### Step 3: copy user's config file, if necessary 130 | 131 | 132 | More practical example: 133 | 134 | ```yaml 135 | --- 136 | # file: complex-playbook.yml 137 | 138 | - hosts: all 139 | 140 | roles: 141 | - williamyeh.fluentd 142 | 143 | vars: 144 | tdagent_plugins: 145 | - fluent-plugin-multiprocess 146 | - fluent-plugin-flowcounter 147 | - fluent-plugin-elasticsearch 148 | 149 | # copy verbatim 150 | tdagent_conf_copy: "files/td-agent.conf" 151 | 152 | # copy through Ansible's template system 153 | tdagent_conf_template: "templates/td-agent.conf.j2" 154 | 155 | # other configurations to be copied through Ansible's template system 156 | tdagent_conf_others: 157 | prometheus_metrics: 158 | src: templates/prometheus.conf.j2 159 | dest: prometheus.conf 160 | ``` 161 | 162 | 163 | ## Dependencies 164 | 165 | None. 166 | 167 | 168 | ## License 169 | 170 | Licensed under the MIT License. See the [LICENSE file](LICENSE) for details. 171 | 172 | 173 | ## History 174 | 175 | Modified from my Dockerized Fluentd app: 176 | 177 | - [williamyeh/fluentd](https://github.com/William-Yeh/docker-fluentd) 178 | --------------------------------------------------------------------------------