├── README.md ├── roles ├── elasticsearch │ └── tasks │ │ └── main.yml ├── filebeat-logzio │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── filebeat.yml.j2 ├── filebeat │ └── tasks │ │ └── main.yml ├── java │ └── tasks │ │ └── main.yml ├── kibana │ └── tasks │ │ └── main.yml └── metricbeat │ └── tasks │ └── main.yml └── site.yml /README.md: -------------------------------------------------------------------------------- 1 | # Ansible ELK Playbook 2 | 3 | This playbook is for setting up version 5.x of the ELK Stack on a remote server. 4 | 5 | ## Notes and requirements 6 | 7 | - The playbook was built and tested on Ubuntu 16.04 VMs, for ELK versions 5.x 8 | - You will need Ansible installed and running 9 | - Playbook is currently configured to set up the ELK stack together with Metricbeat for server perf monitoring. There is a role for Filebeat as well. You just need to add the Filebeat role to your [site.yml] file. 10 | 11 | ## Instructions 12 | 13 | 1. Edit your Ansible hosts file ('/etc/ansible/hosts') and add an 'elkservers' entry for the server you wish to install ELK on. You can of course name the host any way you want, this is just an example. 14 | 2. Verify connectivity to the ELK server. 15 | 3. In the terminal on the machine hosting Ansible, clone this repo. 16 | 4. Cd into the directory, and run: 17 | `ansible-playbook site.yml` 18 | 19 | The plays in the playbook will run on the target server, installing ELK and the specified beats shippers. 20 | 21 | [site.yml]: https://github.com/DanielBerman/ansible-elk-playbook/blob/master/site.yml 22 | -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Installing Elasticsearch 4 | # 5 | 6 | # Add Elasticsearch apt key 7 | 8 | - name: Add Elasticsearch apt key 9 | apt_key: 10 | url: "https://packages.elastic.co/GPG-KEY-elasticsearch" 11 | state: present 12 | 13 | # Add the Elasticsearch apt repo. For versions 6 of the stack - use '6.x-prerelease': 14 | 15 | - name: Adding Elasticsearch repo 16 | apt_repository: 17 | repo: deb https://artifacts.elastic.co/packages/5.x/apt stable main 18 | state: present 19 | 20 | # Installing Elasticsearch 21 | 22 | - name: Install Elasticsearch 23 | apt: 24 | name: elasticsearch 25 | update_cache: yes 26 | 27 | # Update Elasticsearch config file to allow access (to secure Elasticsearch, bind to 'localhost'). 28 | 29 | - name: Updating the config file to allow outside access 30 | lineinfile: 31 | destfile: /etc/elasticsearch/elasticsearch.yml 32 | regexp: 'network.host:' 33 | line: 'network.host: 0.0.0.0' 34 | 35 | # Update Elasticsearch port in config file 36 | 37 | - name: Updating the port in config file 38 | lineinfile: 39 | destfile: /etc/elasticsearch/elasticsearch.yml 40 | regexp: 'http.port:' 41 | line: 'http.port: 9200' 42 | 43 | # Start Elasticsearch 44 | - name: Starting Elasticsearch 45 | service: 46 | name: elasticsearch 47 | state: started 48 | 49 | -------------------------------------------------------------------------------- /roles/filebeat-logzio/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | filebeat_create_config: true 3 | 4 | filebeat_prospectors: 5 | - input_type: log 6 | paths: 7 | - "/var/log/*.log" 8 | fields: 9 | logzio_codec: plain 10 | token: token 11 | files_under_root: true 12 | ignore_older: 3h 13 | 14 | filebeat_output_elasticsearch_enabled: false 15 | filebeat_output_elasticsearch_hosts: 16 | - "localhost:9200" 17 | 18 | filebeat_output_logstash_enabled: true 19 | filebeat_output_logstash_hosts: 20 | - "listener.logz.io:5015" 21 | 22 | filebeat_enable_logging: false 23 | filebeat_log_level: warning 24 | filebeat_log_dir: /var/log/mybeat 25 | filebeat_log_filename: mybeat.log 26 | 27 | filebeat_ssl_dir: /etc/pki/tls/certs 28 | filebeat_ssl_certificate_file: "etc/pki/tls/certs/COMODORSADomainValidationSecureServerCA.crt" 29 | filebeat_ssl_key_file: "" 30 | filebeat_ssl_insecure: "false" 31 | -------------------------------------------------------------------------------- /roles/filebeat-logzio/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Installing Filebeat 4 | # 5 | 6 | # Download SSL certificate: 7 | 8 | - name: Download certificate 9 | shell: wget https://raw.githubusercontent.com/logzio/public-certificates/master/COMODORSADomainValidationSecureServerCA.crt 10 | 11 | # Create directory for certification 12 | 13 | - name: Make new directory for cert 14 | shell: mkdir -p /etc/pki/tls/certs 15 | 16 | # Move cert to directory 17 | 18 | - name: Move SSL certification to new folder 19 | shell: cp COMODORSADomainValidationSecureServerCA.crt /etc/pki/tls/certs/ 20 | 21 | # Install Filebeat 22 | 23 | - name: Install Filebeat with apt 24 | apt: 25 | name: filebeat 26 | update_cache: yes 27 | 28 | # Replace Filebeat configuration to ship to Logz.io. 29 | 30 | - name: Replace default filebeat.yml configurations 31 | template: 32 | src: filebeat.yml.j2 33 | dest: /etc/filebeat/filebeat.yml 34 | 35 | # Starting Filebeat 36 | 37 | - name: Starting Filebeat 38 | service: 39 | name: filebeat 40 | state: started 41 | -------------------------------------------------------------------------------- /roles/filebeat-logzio/templates/filebeat.yml.j2: -------------------------------------------------------------------------------- 1 | filebeat: 2 | # List of prospectors to fetch data. 3 | prospectors: 4 | {{ filebeat_prospectors | to_json }} 5 | 6 | # Configure what outputs to use when sending the data collected by the beat. 7 | # Multiple outputs may be used. 8 | output: 9 | 10 | {% if filebeat_output_elasticsearch_enabled %} 11 | ### Elasticsearch as output 12 | elasticsearch: 13 | # Array of hosts to connect to. 14 | hosts: {{ filebeat_output_elasticsearch_hosts | to_json }} 15 | 16 | # Optional protocol and basic auth credentials. These are deprecated. 17 | #protocol: "https" 18 | #username: "admin" 19 | #password: "s3cr3t" 20 | 21 | # Number of workers per Elasticsearch host. 22 | #worker: 1 23 | 24 | # Optional index name. The default is "filebeat" and generates 25 | # [filebeat-]YYYY.MM.DD keys. 26 | #index: "filebeat" 27 | 28 | # Optional HTTP Path 29 | #path: "/elasticsearch" 30 | 31 | # Proxy server URL 32 | # proxy_url: http://proxy:3128 33 | 34 | # The number of times a particular Elasticsearch index operation is attempted. If 35 | # the indexing operation doesn't succeed after this many retries, the events are 36 | # dropped. The default is 3. 37 | #max_retries: 3 38 | 39 | # The maximum number of events to bulk in a single Elasticsearch bulk API index request. 40 | # The default is 50. 41 | #bulk_max_size: 50 42 | 43 | # Configure http request timeout before failing an request to Elasticsearch. 44 | #timeout: 90 45 | 46 | # The number of seconds to wait for new events between two bulk API index requests. 47 | # If `bulk_max_size` is reached before this interval expires, addition bulk index 48 | # requests are made. 49 | #flush_interval: 1 50 | 51 | # Boolean that sets if the topology is kept in Elasticsearch. The default is 52 | # false. This option makes sense only for Packetbeat. 53 | #save_topology: false 54 | 55 | # The time to live in seconds for the topology information that is stored in 56 | # Elasticsearch. The default is 15 seconds. 57 | #topology_expire: 15 58 | 59 | {% if filebeat_ssl_certificate_file and filebeat_ssl_key_file %} 60 | # tls configuration. By default is off. 61 | tls: 62 | # List of root certificates for HTTPS server verifications 63 | #certificate_authorities: ["/etc/pki/root/ca.pem"] 64 | 65 | # Certificate for TLS client authentication 66 | certificate: "{{ filebeat_ssl_dir }}/{{ filebeat_ssl_certificate_file | basename }}" 67 | 68 | # Client Certificate Key 69 | certificate_key: "{{ filebeat_ssl_dir }}/{{ filebeat_ssl_key_file | basename}}" 70 | 71 | # Controls whether the client verifies server certificates and host name. 72 | # If insecure is set to true, all server host names and certificates will be 73 | # accepted. In this mode TLS based connections are susceptible to 74 | # man-in-the-middle attacks. Use only for testing. 75 | insecure: {{ filebeat_ssl_insecure }} 76 | 77 | # Configure cipher suites to be used for TLS connections 78 | #cipher_suites: [] 79 | 80 | # Configure curve types for ECDHE based cipher suites 81 | #curve_types: [] 82 | 83 | # Configure minimum TLS version allowed for connection to logstash 84 | #min_version: 1.0 85 | 86 | # Configure maximum TLS version allowed for connection to logstash 87 | #max_version: 1.2 88 | {% endif %} 89 | {% endif %} 90 | 91 | {% if filebeat_output_logstash_enabled %} 92 | ### Logstash as output 93 | logstash: 94 | # The Logstash hosts 95 | hosts: {{ filebeat_output_logstash_hosts | to_json }} 96 | 97 | # Number of workers per Logstash host. 98 | #worker: 1 99 | 100 | # Optional load balance the events between the Logstash hosts 101 | #loadbalance: true 102 | 103 | # Optional index name. The default index name depends on the each beat. 104 | # For Packetbeat, the default is set to packetbeat, for Topbeat 105 | # top topbeat and for Filebeat to filebeat. 106 | #index: filebeat 107 | 108 | {% if filebeat_ssl_certificate_file and filebeat_ssl_key_file %} 109 | # Optional TLS. By default is off. 110 | tls: 111 | # List of root certificates for HTTPS server verifications 112 | #certificate_authorities: ["/etc/pki/root/ca.pem"] 113 | 114 | # Certificate for TLS client authentication 115 | certificate: "{{ filebeat_ssl_dir }}/{{ filebeat_ssl_certificate_file | basename }}" 116 | 117 | # Client Certificate Key 118 | certificate_key: "{{ filebeat_ssl_dir }}/{{ filebeat_ssl_key_file | basename}}" 119 | 120 | # Controls whether the client verifies server certificates and host name. 121 | # If insecure is set to true, all server host names and certificates will be 122 | # accepted. In this mode TLS based connections are susceptible to 123 | # man-in-the-middle attacks. Use only for testing. 124 | #insecure: true 125 | insecure: {{ filebeat_ssl_insecure }} 126 | 127 | # Configure cipher suites to be used for TLS connections 128 | #cipher_suites: [] 129 | 130 | # Configure curve types for ECDHE based cipher suites 131 | #curve_types: [] 132 | {% endif %} 133 | 134 | {% if filebeat_enable_logging %} 135 | logging: 136 | ### Filebeat log 137 | level: {{ filebeat_log_level }} 138 | 139 | # Enable file rotation with default configuration 140 | to_files: true 141 | 142 | # Do not log to syslog 143 | to_syslog: false 144 | 145 | files: 146 | path: {{ filebeat_log_dir }} 147 | name: {{ filebeat_log_filename }} 148 | keepfiles: 7 149 | {% endif %} 150 | {% endif %} 151 | -------------------------------------------------------------------------------- /roles/filebeat/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Installing Filebeat 4 | # 5 | 6 | # Install Filebeat 7 | 8 | - name: Install Filebeat with apt 9 | apt: 10 | name: filebeat 11 | update_cache: yes 12 | 13 | # Starting Filebeat 14 | 15 | - name: Starting Filebeat 16 | service: 17 | name: filebeat 18 | state: started 19 | -------------------------------------------------------------------------------- /roles/java/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Installing Java 8 4 | # 5 | 6 | # Add the Java repo 7 | 8 | - name: Add the Java PPA repo 9 | apt_repository: 10 | repo: ppa:webupd8team/java 11 | 12 | # Accept Oracle license 13 | 14 | - name: Automatically accept the Oracle license 15 | shell: echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections 16 | 17 | # Install Java 18 | 19 | - name: Install Java 8 20 | apt: 21 | name: oracle-java8-installer 22 | state: present 23 | update_cache: yes 24 | -------------------------------------------------------------------------------- /roles/kibana/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Installing Kibana 4 | # 5 | 6 | # Install Kibana 7 | 8 | - name: Install Kibana with apt 9 | apt: 10 | name: kibana 11 | update_cache: yes 12 | 13 | # Configurations 14 | 15 | - name: Updating the config file to allow outside access 16 | lineinfile: 17 | destfile: /etc/kibana/kibana.yml 18 | regexp: 'server.host:' 19 | line: 'server.host: 0.0.0.0' 20 | 21 | - name: Defining server port 22 | lineinfile: 23 | destfile: /etc/kibana/kibana.yml 24 | regexp: 'server.port:' 25 | line: 'server.port: 5601' 26 | 27 | - name: Defining Elasticsearch URL 28 | lineinfile: 29 | destfile: /etc/kibana/kibana.yml 30 | regexp: 'elasticsearch.url:' 31 | line: 'elasticsearch.url: "http://localhost:9200"' 32 | 33 | # Starting Kibana 34 | 35 | - name: Starting Kibana 36 | service: 37 | name: kibana 38 | state: started 39 | -------------------------------------------------------------------------------- /roles/metricbeat/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Installing Metricbeat 4 | # 5 | 6 | # Install Metricbeat 7 | 8 | - name: Install Metricbeat with apt 9 | apt: 10 | name: metricbeat 11 | update_cache: yes 12 | 13 | # Starting Metricbeat 14 | 15 | - name: Starting Metricbeat 16 | service: 17 | name: metricbeat 18 | state: started 19 | -------------------------------------------------------------------------------- /site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Playbook to install the ELK stack + Beats 4 | # 5 | - hosts: elkservers 6 | remote_user: ubuntu 7 | become: yes 8 | become_user: root 9 | roles: 10 | - { role: java } 11 | - { role: elasticsearch } 12 | - { role: kibana } 13 | - { role: metricbeat } 14 | 15 | --------------------------------------------------------------------------------