├── ansible.cfg ├── .gitignore ├── vars └── main.yml ├── templates ├── defaults.conf.j2 ├── input.conf.j2 ├── output.conf.j2 ├── filters.conf.j2 └── logstash_repo.j2 ├── handlers └── main.yml ├── tasks ├── logstash_requirements.yml ├── plugins.yml ├── main.yml ├── logstash_installation_RedHat.yml ├── logstash_installation_Debian.yml └── logstash_configuration.yml ├── .travis.yml ├── test.yml ├── defaults └── main.yml ├── meta └── main.yml └── README.md /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | roles_path=../ 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | devel.* 2 | *.devel 3 | inventory.ini 4 | -------------------------------------------------------------------------------- /vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for logstash-role 3 | -------------------------------------------------------------------------------- /templates/defaults.conf.j2: -------------------------------------------------------------------------------- 1 | {% if logstash_defaults is defined %} 2 | {{ logstash_defaults }} 3 | {% endif %} 4 | -------------------------------------------------------------------------------- /templates/input.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | input { 4 | 5 | {{ logstash_inputs }} 6 | 7 | } 8 | -------------------------------------------------------------------------------- /templates/output.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | output { 4 | 5 | {{ logstash_outputs }} 6 | 7 | } 8 | -------------------------------------------------------------------------------- /templates/filters.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | filter { 4 | 5 | {{ logstash_filters }} 6 | 7 | } 8 | -------------------------------------------------------------------------------- /handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for logstash-role 3 | 4 | - name: restart logstash 5 | service: name=logstash 6 | state=restarted 7 | enabled=yes 8 | -------------------------------------------------------------------------------- /tasks/logstash_requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Python utils on Debian OS Family 3 | apt: pkg={{ item.package }} 4 | update_cache=yes 5 | state=latest 6 | with_items: logstash_python_utils 7 | when: ansible_os_family == "Debian" 8 | -------------------------------------------------------------------------------- /templates/logstash_repo.j2: -------------------------------------------------------------------------------- 1 | [logstash-{{ logstash_version }}] 2 | name=logstash repository for {{ logstash_version }}.x packages 3 | baseurl=http://packages.elasticsearch.org/logstash/{{ logstash_version }}/centos 4 | gpgcheck=1 5 | gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch 6 | enabled=1 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | before_install: 5 | - sudo apt-get update -qq 6 | - sudo apt-get install -qq python-apt python-pycurl openjdk-7-jre-headless 7 | install: 8 | - pip install ansible 9 | script: 10 | - echo localhost > inventory 11 | - ansible-playbook --syntax-check -i inventory test.yml 12 | - ansible-playbook -i inventory test.yml --connection=local --sudo 13 | -------------------------------------------------------------------------------- /test.yml: -------------------------------------------------------------------------------- 1 | - hosts: localhost 2 | sudo: yes 3 | roles: 4 | - role: logstash-role 5 | logstash_version: "1.5" 6 | logstash_defaults: | 7 | LS_USER=root 8 | LS_HEAP_SIZE="256m" 9 | 10 | logstash_inputs: | 11 | file { 12 | path => "/var/log/auth.log" 13 | } 14 | 15 | logstash_filters: | 16 | # dummy 17 | 18 | logstash_outputs: | 19 | # dummy 20 | 21 | tags: logstash 22 | -------------------------------------------------------------------------------- /tasks/plugins.yml: -------------------------------------------------------------------------------- 1 | # Loop though logstash_plugins and install them 2 | - name: Removing Plugins if they exist 3 | action: > 4 | shell bin/plugin uninstall "{{ item.name }}" 5 | chdir="{{ logstash_home_dir }}" 6 | with_items: logstash_plugins 7 | ignore_errors: yes 8 | - name: Installing Plugins by Name 9 | action: > 10 | shell bin/plugin install "{{ item.name }}" 11 | chdir="{{ logstash_home_dir }}" 12 | with_items: logstash_plugins 13 | ignore_errors: no 14 | -------------------------------------------------------------------------------- /tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for logstash-role 3 | 4 | - include: logstash_requirements.yml 5 | tags: 6 | - logstash_req 7 | 8 | - include: logstash_installation_Debian.yml 9 | when: ansible_os_family == "Debian" 10 | 11 | - include: logstash_installation_RedHat.yml 12 | when: ansible_os_family == "RedHat" 13 | # Install Other Generic Plugins 14 | - include: plugins.yml 15 | tags: 16 | - logstash_plugins 17 | when: (logstash_plugins is defined) 18 | 19 | - include: logstash_configuration.yml 20 | tags: 21 | - logstash_config 22 | -------------------------------------------------------------------------------- /defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for logstash-role 3 | 4 | logstash_python_utils: 5 | - { package: "python-pycurl" } 6 | - { package: "python-apt" } 7 | 8 | logstash_version: "1.5" 9 | 10 | logstash_apt_repo: "deb http://packages.elasticsearch.org/logstash/{{ logstash_version }}/debian stable main" 11 | logstash_repo_key: "http://packages.elasticsearch.org/GPG-KEY-elasticsearch" 12 | logstash_yum_repo_dest: "/etc/yum.repos.d/logstash.repo" 13 | 14 | logstash_conf_dir: "/etc/logstash/conf.d/" 15 | 16 | logstash_defaults: "LS_USER=logstash" 17 | 18 | defaults_RedHat: "/etc/sysconfig/logstash" 19 | defaults_Debian: "/etc/default/logstash" 20 | logstash_home_dir: "/opt/logstash/" -------------------------------------------------------------------------------- /tasks/logstash_installation_RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Enable Logstash repository 3 | template: src=logstash_repo.j2 4 | dest={{ logstash_yum_repo_dest }} 5 | tags: logstash_repo 6 | 7 | - name: Add Logstash Repo Key 8 | rpm_key: key="{{ logstash_repo_key }}" 9 | state=present 10 | tags: logstash_repo 11 | 12 | - name: Install Logstash 13 | yum: name=logstash 14 | state=latest 15 | tags: logstash_install 16 | 17 | - name: Configure default settings for Logstash 18 | template: src=defaults.conf.j2 19 | dest={{ defaults_RedHat }} 20 | owner=root 21 | group=root 22 | mode=0644 23 | backup=yes 24 | notify: 25 | - restart logstash 26 | tags: 27 | - logstash_defaults 28 | -------------------------------------------------------------------------------- /tasks/logstash_installation_Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Enable Logstash Repository 3 | apt_repository: repo="{{ logstash_apt_repo }}" 4 | state=present 5 | tags: logstash_repo 6 | 7 | - name: Add Logstash Repo Key 8 | apt_key: url="{{ logstash_repo_key }}" 9 | state=present 10 | tags: logstash_repo 11 | 12 | - name: Install Logstash 13 | apt: pkg=logstash 14 | update_cache=yes 15 | state=latest 16 | tags: logstash_install 17 | 18 | - debug: var=logstash_defaults 19 | 20 | - name: Configure default settings for Logstash (Debian only. Upstart on Ubuntu won't be touched) 21 | template: src=defaults.conf.j2 22 | dest={{ defaults_Debian }} 23 | owner=root 24 | group=root 25 | mode=0644 26 | backup=yes 27 | # when: ansible_distribution == "Debian" 28 | notify: 29 | - restart logstash 30 | tags: 31 | - logstash_defaults 32 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Valentino Gagliardi 4 | description: Ansible Role to Install and Configure Logstash 5 | company: ServerManaged.it 6 | # Some suggested licenses: 7 | # - BSD (default) 8 | # - MIT 9 | # - GPLv2 10 | # - GPLv3 11 | # - Apache 12 | # - CC-BY 13 | license: GPLv2 14 | min_ansible_version: 1.7 15 | # 16 | # Below are all platforms currently available. Just uncomment 17 | # the ones that apply to your role. If you don't see your 18 | # platform on this list, let us know and we'll get it added! 19 | # 20 | platforms: 21 | - name: EL 22 | versions: 23 | # - all 24 | # - 5 25 | - 6 26 | - 7 27 | #- name: GenericUNIX 28 | # versions: 29 | # - all 30 | # - any 31 | - name: Fedora 32 | versions: 33 | # - all 34 | # - 16 35 | # - 17 36 | # - 18 37 | # - 19 38 | - 20 39 | #- name: opensuse 40 | # versions: 41 | # - all 42 | # - 12.1 43 | # - 12.2 44 | # - 12.3 45 | # - 13.1 46 | # - 13.2 47 | #- name: Amazon 48 | # versions: 49 | # - all 50 | # - 2013.03 51 | # - 2013.09 52 | #- name: GenericBSD 53 | # versions: 54 | # - all 55 | # - any 56 | #- name: FreeBSD 57 | # versions: 58 | # - all 59 | # - 8.0 60 | # - 8.1 61 | # - 8.2 62 | # - 8.3 63 | # - 8.4 64 | # - 9.0 65 | # - 9.1 66 | # - 9.1 67 | # - 9.2 68 | - name: Ubuntu 69 | versions: 70 | # - all 71 | # - lucid 72 | # - maverick 73 | # - natty 74 | # - oneiric 75 | # - precise 76 | # - quantal 77 | # - raring 78 | # - saucy 79 | - trusty 80 | #- name: SLES 81 | # versions: 82 | # - all 83 | # - 10SP3 84 | # - 10SP4 85 | # - 11 86 | # - 11SP1 87 | # - 11SP2 88 | # - 11SP3 89 | #- name: GenericLinux 90 | # versions: 91 | # - all 92 | # - any 93 | - name: Debian 94 | versions: 95 | # - all 96 | # - etch 97 | # - lenny 98 | # - squeeze 99 | - wheezy 100 | # 101 | # Below are all categories currently available. Just as with 102 | # the platforms above, uncomment those that apply to your role. 103 | # 104 | categories: 105 | #- cloud 106 | #- cloud:ec2 107 | #- cloud:gce 108 | #- cloud:rax 109 | #- clustering 110 | #- database 111 | #- database:nosql 112 | #- database:sql 113 | #- development 114 | - monitoring 115 | #- networking 116 | #- packaging 117 | #- system 118 | #- web 119 | dependencies: [] 120 | # List your role dependencies here, one per line. Only 121 | # dependencies available via galaxy should be listed here. 122 | # Be sure to remove the '[]' above if you add dependencies 123 | # to this list. 124 | 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Ansible Role to Install and Configure Logstash 2 | ========= 3 | 4 | 5 | Requirements 6 | ------------ 7 | 8 | On Debian OS family, **python-pycurl** and **python-apt** are required to deal with apt Ansible modules. The role already take care of these dependencies, you can disable this behavior with `--skip-tags=logstash_req`. 9 | 10 | **Java** should be present on the nodes machines in order to run Logstash. This role does not install Java. 11 | 12 | Logstash configuration can be set in playbook input, or in a folder with files. 13 | The files will be run through the templating engine - meaning you can use variables. 14 | 15 | 16 | Example Playbooks 17 | ---------------- 18 | 19 | ```yaml 20 | - hosts: LogstashNodes 21 | roles: 22 | - role: logstash-role 23 | 24 | logstash_version: "1.5" 25 | 26 | logstash_defaults: | 27 | LS_USER=root 28 | LS_HEAP_SIZE="256m" 29 | 30 | logstash_config_files: include/logstash 31 | 32 | logstash_inputs: | 33 | syslog { host => "{{ ansible_eth0.ipv4.address }}" 34 | port => "514" 35 | type => "syslog_input" 36 | } 37 | 38 | syslog { host => "{{ ansible_lo.ipv4.address }}" 39 | port => "515" 40 | type => "syslog_input_local" 41 | } 42 | 43 | logstash_filters: | 44 | geoip { source => "ip_address" 45 | } 46 | 47 | multiline { pattern => "^No lfn2pfn" 48 | what => "previous" 49 | } 50 | 51 | logstash_outputs: | 52 | file { path => "/var/log/logstash/output.log" 53 | } 54 | 55 | tags: logstash 56 | ``` 57 | 58 | Role Variables 59 | -------------- 60 | 61 | ```yaml 62 | logstash_python_utils: 63 | - { package: "python-pycurl" } 64 | - { package: "python-apt" } 65 | 66 | logstash_version: "none" 67 | 68 | logstash_apt_repo: "deb http://packages.elasticsearch.org/logstash/{{ logstash_version }}/debian stable main" 69 | logstash_repo_key: "http://packages.elasticsearch.org/GPG-KEY-elasticsearch" 70 | logstash_yum_repo_dest: "/etc/yum.repos.d/logstash.repo" 71 | 72 | logstash_conf_dir: "/etc/logstash/conf.d/" 73 | 74 | logstash_defaults: 75 | - { directive: "LS_USER=logstash" } 76 | 77 | defaults_RedHat: "/etc/sysconfig/logstash" 78 | defaults_Debian: "/etc/default/logstash" 79 | ``` 80 | 81 | 82 | 83 | 84 | ## Include role in a larger playbook 85 | ### Add this role as a git submodule 86 | Assuming your playbook structure is such as: 87 | ``` 88 | - my-master-playbook 89 | |- vars 90 | |- roles 91 | |- my-master-playbook-main.yml 92 | \- my-master-inventory.ini 93 | ``` 94 | 95 | Checkout this project as a submodule under roles: 96 | 97 | ``` 98 | $ cd roles 99 | $ git submodule add https://github.com/comperiosearch/logstash-role.git ./logstash 100 | $ git submodule update --init 101 | $ git commit ./submodule -m "Added submodule as ./subm" 102 | ``` 103 | 104 | ### Include this playbook as a role in your master playbook 105 | Example `my-master-playbook-main.yml`: 106 | 107 | ``` 108 | --- 109 | 110 | ######################### 111 | # Logstash install # 112 | ######################### 113 | 114 | - hosts: all_nodes 115 | user: ubuntu 116 | sudo: yes 117 | 118 | roles: 119 | - logstash 120 | 121 | vars_files: 122 | - vars/my-vars.yml 123 | ``` 124 | 125 | 126 | License 127 | ------- 128 | 129 | GNU General Public License Version 2 130 | 131 | Author Information 132 | ------------------ 133 | 134 | Valentino Gagliardi - valentino.g@servermanaged.it 135 | Christoffer Vig - christoffer.vig@gmail.com 136 | -------------------------------------------------------------------------------- /tasks/logstash_configuration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #TODO should delete all existing configs -but only if successful valitation.... 3 | - name: get the list of input templates to transfer 4 | local_action: "shell ls {{logstash_config_files_input}}/* | sed 's~.*/~~g'" 5 | tags: logstash_config_files 6 | register: template_files_input 7 | when: logstash_config_files_input is defined 8 | 9 | - name: get the list of filter templates to transfer 10 | local_action: "shell ls {{logstash_config_files_filter}}/* | sed 's~.*/~~g'" 11 | tags: logstash_config_files 12 | register: template_files_filter 13 | when: logstash_config_files_filter is defined 14 | 15 | - name: get the list of output templates to transfer 16 | local_action: "shell ls {{logstash_config_files_output}}/* | sed 's~.*/~~g'" 17 | tags: logstash_config_files 18 | register: template_files_output 19 | when: logstash_config_files_output is defined 20 | 21 | - name: Copy Logstash input config files 22 | template: src={{logstash_config_files_input}}/{{ item }} 23 | dest={{ logstash_conf_dir }}/{{ item }} 24 | #validate="{{ logstash_home_dir }}/bin/logstash agent -t -f %s" 25 | with_items: 26 | - "{{ template_files_input.stdout.splitlines() }}" 27 | notify: 28 | - restart logstash 29 | tags: logstash_config_files 30 | when: logstash_config_files_input is defined 31 | 32 | 33 | 34 | - name: Copy Logstash filter config files 35 | template: src={{logstash_config_files_filter}}/{{ item }} 36 | dest={{ logstash_conf_dir }}/{{ item }} 37 | #validate="{{ logstash_home_dir }}/bin/logstash agent -t -f %s" 38 | with_items: 39 | - "{{ template_files_filter.stdout.splitlines() }}" 40 | notify: 41 | - restart logstash 42 | tags: logstash_config_files 43 | when: logstash_config_files_filter is defined 44 | 45 | 46 | 47 | - name: Copy Logstash output config files 48 | template: src={{logstash_config_files_output}}/{{ item }} 49 | dest={{ logstash_conf_dir }}/{{ item }} 50 | #validate="{{ logstash_home_dir }}/bin/logstash agent -t -f %s" 51 | with_items: 52 | - "{{ template_files_output.stdout.splitlines() }}" 53 | notify: 54 | - restart logstash 55 | tags: logstash_config_files 56 | when: logstash_config_files_output is defined 57 | 58 | 59 | 60 | # cLEAN AND DELETE UNMANGED FILES 61 | 62 | # - shell: ls -1 {{ logstash_conf_dir }} 63 | # register: logstash_conf_contents 64 | # tags: logstash_config_files 65 | 66 | # - name: Removing unmanaged files from logstash config dir 67 | # file: path={{ logstash_conf_dir }}/{{ item }} state=absent 68 | # with_items: logstash_conf_contents.stdout_lines 69 | # when: item not in (( template_files_input.stdout.splitlines() + template_files_filter.stdout.splitlines() + template_files_output.stdout.splitlines()) + ["00-ansible-gen-input.conf" ,"51-ansible-gen-filters.conf", "99-ansible-gen-output.conf" ]) 70 | # tags: logstash_config_files 71 | # when: (logstash_config_files_input is defined or logstash_config_files_filter is defined or logstash_config_files_output is defined ) 72 | 73 | - name: Copy Logstash Input Configuration 74 | template: src=input.conf.j2 75 | dest={{ logstash_conf_dir }}00-ansible-gen-input.conf 76 | owner=root 77 | group=root 78 | mode=0644 79 | #validate="{{ logstash_home_dir }}/bin/logstash agent -t -f %s" 80 | notify: 81 | - restart logstash 82 | tags: logstash_inputs 83 | when: logstash_inputs is defined 84 | 85 | - name: Copy Logstash Filter Configuration 86 | template: src=filters.conf.j2 87 | dest={{ logstash_conf_dir }}51-ansible-gen-filters.conf 88 | owner=root 89 | group=root 90 | mode=0644 91 | #validate="{{ logstash_home_dir }}/bin/logstash agent -t -f %s" 92 | notify: 93 | - restart logstash 94 | tags: logstash_filters 95 | when: logstash_filters is defined 96 | 97 | - name: Copy Logstash Output Configuration 98 | template: src=output.conf.j2 99 | dest={{ logstash_conf_dir }}99-ansible-gen-output.conf 100 | owner=root 101 | group=root 102 | mode=0644 103 | #validate="{{ logstash_home_dir }}/bin/logstash agent -t -f %s" 104 | notify: 105 | - restart logstash 106 | tags: logstash_outputs 107 | when: logstash_outputs is defined 108 | --------------------------------------------------------------------------------