├── .ruby-version ├── ansible.cfg ├── test ├── integration │ ├── oss │ │ ├── oss.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── trial │ │ ├── trial.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── default │ │ ├── default.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── license │ │ ├── license.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── oss-upgrade │ │ ├── oss.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── security │ │ ├── security.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── upgrade │ │ ├── upgrade.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── custom-config │ │ ├── custom_config.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── oss-to-default-upgrade │ │ ├── oss-to-default-upgrade.yml │ │ └── serverspec │ │ │ └── default_spec.rb │ ├── helpers │ │ └── serverspec │ │ │ ├── Gemfile │ │ │ ├── custom_config_spec.rb │ │ │ ├── spec_helper.rb │ │ │ ├── license_spec.rb │ │ │ ├── security_spec.rb │ │ │ └── shared_spec.rb │ ├── files │ │ ├── certs │ │ │ ├── keystore-password.p12 │ │ │ ├── keystore-password-ca.p12 │ │ │ ├── truststore-password.p12 │ │ │ ├── truststore-password-ca.p12 │ │ │ ├── shared-store-no-password.p12 │ │ │ └── shared-store-no-password-ca.p12 │ │ ├── templates-7.x │ │ │ └── basic.json │ │ ├── custom_config │ │ │ ├── elasticsearch │ │ │ ├── jvm.options │ │ │ └── log4j2.properties │ │ └── templates-6.x │ │ │ └── basic.json │ ├── debug.yml │ ├── default.yml │ ├── oss.yml │ ├── custom-config.yml │ ├── upgrade.yml │ ├── oss-upgrade.yml │ ├── oss-to-default-upgrade.yml │ ├── security.yml │ ├── license.yml │ └── trial.yml ├── matrix.yml └── matrix-6x.yml ├── templates ├── security │ ├── users_roles.j2 │ ├── roles.yml.j2 │ └── role_mapping.yml.j2 ├── override.conf.j2 ├── elasticsearch.repo ├── elasticsearch.j2 ├── elasticsearch.yml.j2 └── jvm.options.j2 ├── files └── systemd │ └── elasticsearch_override.conf ├── Gemfile ├── .gitignore ├── vars ├── RedHat.yml ├── main.yml └── Debian.yml ├── tasks ├── elasticsearch.yml ├── xpack │ ├── elasticsearch-xpack.yml │ └── security │ │ ├── elasticsearch-xpack-trial-activation.yml │ │ ├── elasticsearch-xpack-activation.yml │ │ ├── elasticsearch-security.yml │ │ ├── elasticsearch-security-file.yml │ │ └── elasticsearch-security-native.yml ├── elasticsearch-optional-user.yml ├── elasticsearch-template.yml ├── elasticsearch-RedHat-version-lock.yml ├── java.yml ├── compatibility-variables.yml ├── snapshot-release.yml ├── elasticsearch-RedHat.yml ├── elasticsearch-config.yml ├── elasticsearch-parameters.yml ├── main.yml ├── elasticsearch-plugins.yml ├── elasticsearch-Debian.yml └── elasticsearch-ssl.yml ├── handlers └── main.yml ├── meta └── main.yml ├── LICENSE ├── Makefile ├── .ci └── jobs │ ├── elastic+ansible-elasticsearch+pull-request.yml │ ├── elastic+ansible-elasticsearch+pull-request-6x.yml │ ├── elastic+ansible-elasticsearch+main.yml │ ├── elastic+ansible-elasticsearch+main-6x.yml │ └── defaults.yml ├── .github ├── issue_template.md └── stale.yml ├── filter_plugins └── custom.py ├── defaults └── main.yml ├── Gemfile.lock ├── docs ├── ssl-tls-setup.md ├── 8x-support.md └── multi-instance.md ├── .kitchen.yml ├── README.md └── CHANGELOG.md /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.5.7 2 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | -------------------------------------------------------------------------------- /test/integration/oss/oss.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /templates/security/users_roles.j2: -------------------------------------------------------------------------------- 1 | {{ users_roles | join("\n") }} 2 | -------------------------------------------------------------------------------- /test/integration/trial/trial.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /test/integration/default/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /test/integration/license/license.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /test/integration/oss-upgrade/oss.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /test/integration/security/security.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /test/integration/upgrade/upgrade.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /files/systemd/elasticsearch_override.conf: -------------------------------------------------------------------------------- 1 | [Service] 2 | LimitMEMLOCK=infinity 3 | -------------------------------------------------------------------------------- /templates/security/roles.yml.j2: -------------------------------------------------------------------------------- 1 | {{ es_roles.file | to_nice_yaml(indent=2) }} 2 | -------------------------------------------------------------------------------- /test/integration/custom-config/custom_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /templates/security/role_mapping.yml.j2: -------------------------------------------------------------------------------- 1 | {{ es_role_mapping | to_nice_yaml(indent=2) }} 2 | -------------------------------------------------------------------------------- /test/integration/oss-to-default-upgrade/oss-to-default-upgrade.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'rspec-retry' 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'test-kitchen' 4 | gem 'kitchen-docker' 5 | gem 'kitchen-ansible' 6 | gem 'net-ssh' 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .kitchen/ 2 | license*.json 3 | *.pyc 4 | .vendor 5 | .bundle 6 | Converging 7 | TODO 8 | .idea/ 9 | elasticsearch.iml 10 | !/vars/RedHat.yml 11 | -------------------------------------------------------------------------------- /test/integration/files/certs/keystore-password.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ansible-elasticsearch/main/test/integration/files/certs/keystore-password.p12 -------------------------------------------------------------------------------- /test/integration/files/certs/keystore-password-ca.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ansible-elasticsearch/main/test/integration/files/certs/keystore-password-ca.p12 -------------------------------------------------------------------------------- /test/integration/files/certs/truststore-password.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ansible-elasticsearch/main/test/integration/files/certs/truststore-password.p12 -------------------------------------------------------------------------------- /test/integration/files/certs/truststore-password-ca.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ansible-elasticsearch/main/test/integration/files/certs/truststore-password-ca.p12 -------------------------------------------------------------------------------- /vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" 3 | default_file: "/etc/sysconfig/elasticsearch" 4 | es_home: "/usr/share/elasticsearch" 5 | -------------------------------------------------------------------------------- /test/integration/files/certs/shared-store-no-password.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ansible-elasticsearch/main/test/integration/files/certs/shared-store-no-password.p12 -------------------------------------------------------------------------------- /test/integration/files/certs/shared-store-no-password-ca.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ansible-elasticsearch/main/test/integration/files/certs/shared-store-no-password-ca.p12 -------------------------------------------------------------------------------- /test/integration/debug.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Dump all variables to a file 3 | changed_when: False 4 | copy: 5 | content: '{{ vars | to_nice_json }} ' 6 | dest: '/tmp/vars.json' 7 | -------------------------------------------------------------------------------- /templates/override.conf.j2: -------------------------------------------------------------------------------- 1 | [Service] 2 | {% if m_lock_enabled %} 3 | LimitMEMLOCK = infinity 4 | {% endif %} 5 | {% if es_max_open_files is defined %} 6 | LimitNOFILE = {{ es_max_open_files }} 7 | {% endif %} -------------------------------------------------------------------------------- /test/integration/files/templates-7.x/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "index_patterns" : "te*", 3 | "settings" : { 4 | "number_of_shards" : 1 5 | }, 6 | "mappings" : { 7 | "_source" : { "enabled" : false } 8 | } 9 | } -------------------------------------------------------------------------------- /test/integration/oss/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'shared_spec' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | describe 'oss tests' do 6 | include_examples 'shared::init', vars 7 | end 8 | -------------------------------------------------------------------------------- /test/integration/trial/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'shared_spec' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | describe 'trial tests' do 6 | include_examples 'shared::init', vars 7 | end 8 | -------------------------------------------------------------------------------- /test/integration/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Setup Elasticsearch using default configuration 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | -------------------------------------------------------------------------------- /test/integration/default/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'shared_spec' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | describe 'default tests' do 6 | include_examples 'shared::init', vars 7 | end 8 | -------------------------------------------------------------------------------- /test/integration/upgrade/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'shared_spec' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | describe 'upgrade tests' do 6 | include_examples 'shared::init', vars 7 | end 8 | -------------------------------------------------------------------------------- /test/integration/files/custom_config/elasticsearch: -------------------------------------------------------------------------------- 1 | # Elasticsearch CUSTOM FILE 2 | 3 | ES_HOME=/usr/share/elasticsearch 4 | ES_PATH_CONF=/etc/elasticsearch 5 | PID_DIR=/var/run/elasticsearch 6 | ES_JAVA_OPTS= 7 | ES_STARTUP_SLEEP_TIME=5 8 | -------------------------------------------------------------------------------- /test/integration/oss-upgrade/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'shared_spec' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | describe 'oss upgrade tests' do 6 | include_examples 'shared::init', vars 7 | end 8 | -------------------------------------------------------------------------------- /test/integration/files/templates-6.x/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "te*", 3 | "settings" : { 4 | "number_of_shards" : 1 5 | }, 6 | "mappings" : { 7 | "type1" : { 8 | "_source" : { "enabled" : false } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | es_package_url: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch" 3 | reserved_xpack_users: ["elastic","kibana","logstash_system"] 4 | sysd_config_file: "/etc/systemd/system/elasticsearch.service.d/override.conf" 5 | -------------------------------------------------------------------------------- /test/integration/oss-to-default-upgrade/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'shared_spec' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | describe 'oss to default upgrade tests' do 6 | include_examples 'shared::init', vars 7 | end 8 | -------------------------------------------------------------------------------- /tasks/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Include specific Elasticsearch 4 | include: elasticsearch-Debian.yml 5 | when: ansible_os_family == 'Debian' 6 | 7 | - name: Include specific Elasticsearch 8 | include: elasticsearch-RedHat.yml 9 | when: ansible_os_family == 'RedHat' 10 | -------------------------------------------------------------------------------- /test/integration/license/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'license_spec' 2 | require 'shared_spec' 3 | require 'json' 4 | vars = JSON.parse(File.read('/tmp/vars.json')) 5 | 6 | describe 'license tests' do 7 | include_examples 'shared::init', vars 8 | include_examples 'license::init', vars 9 | end 10 | -------------------------------------------------------------------------------- /test/integration/security/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'security_spec' 2 | require 'shared_spec' 3 | require 'json' 4 | vars = JSON.parse(File.read('/tmp/vars.json')) 5 | 6 | describe 'security tests' do 7 | include_examples 'shared::init', vars 8 | include_examples 'security::init', vars 9 | end 10 | -------------------------------------------------------------------------------- /test/integration/custom-config/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'custom_config_spec' 2 | require 'shared_spec' 3 | require 'json' 4 | vars = JSON.parse(File.read('/tmp/vars.json')) 5 | 6 | describe 'Custom Config Tests' do 7 | include_examples 'custom_config::init', vars 8 | include_examples 'shared::init', vars 9 | end 10 | -------------------------------------------------------------------------------- /test/integration/oss.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Setup Elasticsearch OSS 6.x 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | oss_version: true 10 | 11 | #Do not add tests here. This test is run twice and confirms idempotency. 12 | -------------------------------------------------------------------------------- /test/matrix.yml: -------------------------------------------------------------------------------- 1 | OS: 2 | - ubuntu-1604 3 | - ubuntu-1804 4 | - ubuntu-2004 5 | - debian-8 6 | - debian-9 7 | - debian-10 8 | - centos-7 9 | - amazonlinux-2 10 | TEST_TYPE: 11 | - custom-config 12 | - default 13 | - license 14 | - oss-to-default-upgrade 15 | - security 16 | - trial 17 | - upgrade 18 | -------------------------------------------------------------------------------- /test/matrix-6x.yml: -------------------------------------------------------------------------------- 1 | OS: 2 | - ubuntu-1604 3 | - ubuntu-1804 4 | - ubuntu-2004 5 | - debian-8 6 | - debian-9 7 | - debian-10 8 | - centos-7 9 | - amazonlinux-2 10 | TEST_TYPE: 11 | - custom-config 12 | - default 13 | - license 14 | - oss 15 | - oss-to-default-upgrade 16 | - oss-upgrade 17 | - security 18 | - trial 19 | - upgrade 20 | -------------------------------------------------------------------------------- /handlers/main.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: reload systemd configuration 3 | become: yes 4 | systemd: 5 | daemon_reload: true 6 | 7 | # Restart service and ensure it is enabled 8 | 9 | - name: restart elasticsearch 10 | become: yes 11 | service: name=elasticsearch state=restarted enabled=yes 12 | when: 13 | - es_restart_on_change 14 | - es_start_service 15 | register: es_restarted 16 | -------------------------------------------------------------------------------- /vars/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | java_version: "{% if ansible_os_family == 'Debian' and ansible_distribution_major_version == '10' %}11{% else %}8{% endif %}" 3 | java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-{{ java_version }}-jre-headless{% endif %}" 4 | default_file: "/etc/default/elasticsearch" 5 | es_home: "/usr/share/elasticsearch" 6 | es_apt_key_id: "46095ACC8548582C1A2699A9D27D666CD88E42B4" 7 | -------------------------------------------------------------------------------- /templates/elasticsearch.repo: -------------------------------------------------------------------------------- 1 | [elasticsearch-{{ es_repo_name }}] 2 | name=Elasticsearch repository for {{ es_repo_name }} packages 3 | baseurl={{ es_repo_base }}/packages/{{ es_repo_name }}/yum 4 | gpgcheck=1 5 | gpgkey={{ es_repo_base }}/GPG-KEY-elasticsearch 6 | enabled=1 7 | autorefresh=1 8 | type=rpm-md 9 | {% if es_proxy_host is defined and es_proxy_host != '' and es_proxy_port is defined %} 10 | proxy=http://{{ es_proxy_host }}:{{es_proxy_port}} 11 | {% endif %} 12 | -------------------------------------------------------------------------------- /test/integration/custom-config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Setup Elasticsearch with custom config files 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_config_default: "test/integration/files/custom_config/elasticsearch" 10 | es_config_log4j2: "test/integration/files/custom_config/log4j2.properties" 11 | es_config_jvm: "test/integration/files/custom_config/jvm.options" 12 | -------------------------------------------------------------------------------- /test/integration/upgrade.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Deploy Elasticsearch previous version 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" 10 | 11 | - name: Deploy Elasticsearch latest version 12 | hosts: localhost 13 | post_tasks: 14 | - include: elasticsearch/test/integration/debug.yml 15 | roles: 16 | - elasticsearch 17 | -------------------------------------------------------------------------------- /tasks/xpack/elasticsearch-xpack.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | #Security configuration 4 | - name: include security/elasticsearch-security.yml 5 | include: security/elasticsearch-security.yml 6 | when: not oss_version 7 | 8 | #Make sure elasticsearch.keystore has correct Permissions 9 | - name: Set elasticsearch.keystore Permissions 10 | become: yes 11 | file: 12 | state: file 13 | path: "{{ es_conf_dir }}/elasticsearch.keystore" 14 | owner: root 15 | group: "{{ es_group }}" 16 | mode: "0660" 17 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | allow_duplicates: yes 4 | 5 | galaxy_info: 6 | author: Robin Clarke, Jakob Reiter, Dale McDiarmid 7 | description: Elasticsearch for Linux 8 | company: "Elastic.co" 9 | license: "license (Apache)" 10 | min_ansible_version: 2.5.0 11 | platforms: 12 | - name: EL 13 | versions: 14 | - 6 15 | - 7 16 | - name: Debian 17 | versions: 18 | - all 19 | - name: Ubuntu 20 | versions: 21 | - all 22 | - galaxy_tags: 23 | - elastic 24 | - elasticsearch 25 | - elk 26 | - logging 27 | dependencies: [] 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2016 Elasticsearch 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: build 2 | 3 | SHELL:=/bin/bash -eux 4 | export VERSION := 7.x 5 | PATTERN := default-ubuntu-1804 6 | 7 | .PHONY: converge verify test login destroy list 8 | 9 | setup: 10 | bundle install 11 | docker ps 12 | 13 | converge: 14 | bundle exec kitchen converge $(PATTERN) 15 | 16 | verify: 17 | bundle exec kitchen verify $(PATTERN) 18 | 19 | test: 20 | bundle exec kitchen test $(PATTERN) --destroy=always 21 | 22 | login: 23 | bundle exec kitchen login $(PATTERN) 24 | 25 | destroy: 26 | bundle exec kitchen destroy $(PATTERN) 27 | 28 | destroy-all: 29 | bundle exec kitchen destroy 30 | 31 | list: 32 | bundle exec kitchen list 33 | -------------------------------------------------------------------------------- /test/integration/oss-upgrade.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Deploy Elasticsearch OSS previous version 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade 10 | oss_version: true 11 | 12 | - name: Deploy Elasticsearch OSS latest version 13 | hosts: localhost 14 | post_tasks: 15 | - include: elasticsearch/test/integration/debug.yml 16 | roles: 17 | - elasticsearch 18 | vars: 19 | oss_version: true 20 | -------------------------------------------------------------------------------- /test/integration/oss-to-default-upgrade.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Deploy Elasticsearch OSS version 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade 10 | oss_version: true 11 | 12 | - name: Upgrade to Elasticsearch default version 13 | hosts: localhost 14 | post_tasks: 15 | - include: elasticsearch/test/integration/debug.yml 16 | roles: 17 | - elasticsearch 18 | vars: 19 | oss_version: false 20 | -------------------------------------------------------------------------------- /tasks/xpack/security/elasticsearch-xpack-trial-activation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Activate ES trial license (with security authentication) 3 | uri: 4 | method: POST 5 | url: "{{ es_api_uri }}/{{ es_license_api }}/start_trial?acknowledge=true" 6 | user: "{{es_api_basic_auth_username | default(omit)}}" 7 | password: "{{es_api_basic_auth_password | default(omit)}}" 8 | return_content: yes 9 | force_basic_auth: yes 10 | status_code: 11 | - 200 12 | - 403 13 | validate_certs: "{{ es_validate_certs }}" 14 | register: trial_license_activated 15 | when: es_xpack_trial 16 | 17 | - name: Trial license 18 | debug: msg={{ trial_license_activated }} -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/custom_config_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'shared_spec' 3 | 4 | shared_examples 'custom_config::init' do |vars| 5 | describe file("/etc/elasticsearch/log4j2.properties") do 6 | it { should be_file } 7 | it { should be_owned_by 'root' } 8 | it { should contain 'Log4j CUSTOM FILE' } 9 | end 10 | describe file("/etc/elasticsearch/jvm.options") do 11 | it { should be_file } 12 | it { should be_owned_by 'root' } 13 | it { should contain 'JVM configuration CUSTOM FILE' } 14 | end 15 | describe file($family['defaults_path']) do 16 | it { should be_file } 17 | it { should be_owned_by 'root' } 18 | it { should contain 'Elasticsearch CUSTOM FILE' } 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /tasks/elasticsearch-optional-user.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #Add the elasticsearch user before installing from packages. 3 | - name: Ensure optional elasticsearch group is created with the correct id. 4 | become: yes 5 | #Restart if these change 6 | notify: restart elasticsearch 7 | group: 8 | state: present 9 | name: "{{ es_group }}" 10 | system: yes 11 | gid: "{{ es_group_id }}" 12 | 13 | - name: Ensure optional elasticsearch user is created with the correct id. 14 | become: yes 15 | #Restart if these change 16 | notify: restart elasticsearch 17 | user: 18 | state: present 19 | name: "{{ es_user }}" 20 | comment: elasticsearch system user 21 | system: yes 22 | createhome: no 23 | uid: "{{ es_user_id }}" 24 | group: "{{ es_group }}" 25 | -------------------------------------------------------------------------------- /tasks/xpack/security/elasticsearch-xpack-activation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Activate ES license (with security authentication) 3 | uri: 4 | method: PUT 5 | url: "{{ es_api_uri }}/{{ es_license_api }}?acknowledge=true" 6 | user: "{{es_api_basic_auth_username | default(omit)}}" 7 | password: "{{es_api_basic_auth_password | default(omit)}}" 8 | body_format: json 9 | body: "{{ es_xpack_license }}" 10 | return_content: yes 11 | force_basic_auth: yes 12 | validate_certs: "{{ es_validate_certs }}" 13 | register: license_activated 14 | no_log: True 15 | failed_when: > 16 | license_activated.status != 200 or 17 | license_activated.json.license_status is not defined or 18 | license_activated.json.license_status != 'valid' 19 | 20 | - name: License 21 | debug: msg={{ license_activated }} 22 | -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'serverspec' 2 | require 'net/http' 3 | require 'json' 4 | 5 | set :backend, :exec 6 | 7 | require 'rspec/retry' 8 | 9 | RSpec.configure do |config| 10 | # show retry status in spec process 11 | config.verbose_retry = true 12 | # show exception that triggers a retry if verbose_retry is set to true 13 | config.display_try_failure_messages = true 14 | end 15 | 16 | def curl_json(uri, username=nil, password=nil) 17 | uri = URI(uri) 18 | req = Net::HTTP::Get.new(uri) 19 | if username && password 20 | req.basic_auth username, password 21 | end 22 | res = Net::HTTP.start( 23 | uri.hostname, 24 | uri.port, 25 | :use_ssl => uri.scheme == 'https', 26 | :verify_mode => OpenSSL::SSL::VERIFY_NONE 27 | ) {|http| 28 | http.request(req) 29 | } 30 | return JSON.parse(res.body) 31 | end 32 | -------------------------------------------------------------------------------- /tasks/elasticsearch-template.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: ensure templates dir is created 4 | file: 5 | path: "{{ es_conf_dir }}/templates" 6 | state: directory 7 | owner: root 8 | group: "{{ es_group }}" 9 | mode: "2750" 10 | 11 | - name: Copy templates to elasticsearch 12 | copy: src={{ item }} dest={{ es_conf_dir }}/templates owner=root group={{ es_group }} mode=0660 13 | register: load_templates 14 | with_fileglob: 15 | - "{{ es_templates_fileglob | default('') }}" 16 | 17 | - name: Install templates 18 | uri: 19 | url: "{{ es_api_uri }}/_template/{{item | filename}}" 20 | method: PUT 21 | status_code: 200 22 | user: "{{es_api_basic_auth_username | default(omit)}}" 23 | password: "{{es_api_basic_auth_password | default(omit)}}" 24 | force_basic_auth: yes 25 | body_format: json 26 | body: "{{ lookup('file', item) }}" 27 | validate_certs: "{{ es_validate_certs }}" 28 | when: load_templates.changed and es_start_service 29 | with_fileglob: 30 | - "{{ es_templates_fileglob | default('') }}" 31 | run_once: True 32 | -------------------------------------------------------------------------------- /test/integration/files/custom_config/jvm.options: -------------------------------------------------------------------------------- 1 | ## JVM configuration CUSTOM FILE 2 | 3 | -Xms2g 4 | -Xmx2g 5 | -Des.networkaddress.cache.ttl=60 6 | -Des.networkaddress.cache.negative.ttl=10 7 | -XX:+AlwaysPreTouch 8 | -Xss1m 9 | -Djava.awt.headless=true 10 | -Dfile.encoding=UTF-8 11 | -Djna.nosys=true 12 | -XX:-OmitStackTraceInFastThrow 13 | -Dio.netty.noUnsafe=true 14 | -Dio.netty.noKeySetOptimization=true 15 | -Dio.netty.recycler.maxCapacityPerThread=0 16 | -Dlog4j.shutdownHookEnabled=false 17 | -Dlog4j2.disable.jmx=true 18 | -Djava.io.tmpdir=${ES_TMPDIR} 19 | -XX:+HeapDumpOnOutOfMemoryError 20 | -XX:HeapDumpPath=${heap.dump.path} 21 | -XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log 22 | 8:-XX:+PrintGCDetails 23 | 8:-XX:+PrintGCDateStamps 24 | 8:-XX:+PrintTenuringDistribution 25 | 8:-XX:+PrintGCApplicationStoppedTime 26 | 8:-Xloggc:/var/log/elasticsearch/gc.log 27 | 8:-XX:+UseGCLogFileRotation 28 | 8:-XX:NumberOfGCLogFiles=32 29 | 8:-XX:GCLogFileSize=64m 30 | 9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m 31 | 9-:-Djava.locale.providers=COMPAT 32 | -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/license_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'json' 3 | require 'pathname' 4 | vars = JSON.parse(File.read('/tmp/vars.json')) 5 | 6 | es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" 7 | username = vars['es_api_basic_auth_username'] 8 | password = vars['es_api_basic_auth_password'] 9 | es_keystore = Pathname.new(vars['es_ssl_keystore']).basename.to_s 10 | es_truststore = Pathname.new(vars['es_ssl_truststore']).basename.to_s 11 | 12 | if vars['es_major_version'] == '7.x' 13 | es_license_api = "_license" 14 | else 15 | es_license_api = "_xpack/license" 16 | end 17 | 18 | shared_examples 'license::init' do |vars| 19 | describe 'License check' do 20 | result = curl_json("#{es_api_url}/#{es_license_api}", username=username, password=password) 21 | it 'should list the license issued by Elastic' do 22 | expect(result['license']['status']).to eq('active') 23 | expect(result['license']['type']).to eq('trial') 24 | expect(result['license']['issued_to']).to eq('Elastic - INTERNAL (non-production environments)') 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /.ci/jobs/elastic+ansible-elasticsearch+pull-request.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: elastic+ansible-elasticsearch+pull-request 4 | display-name: elastic / ansible-elasticsearch - pull-request 5 | description: Pull request testing with test kitchen 6 | parameters: 7 | - string: 8 | name: VERSION 9 | default: 7.x 10 | description: Elasticsearch major version 11 | axes: 12 | - axis: 13 | type: slave 14 | name: label 15 | values: 16 | - linux 17 | - axis: 18 | name: OS 19 | filename: elasticsearch/test/matrix.yml 20 | type: yaml 21 | - axis: 22 | name: TEST_TYPE 23 | filename: elasticsearch/test/matrix.yml 24 | type: yaml 25 | scm: 26 | - git: 27 | branches: 28 | - $ghprbActualCommit 29 | refspec: +refs/pull/*:refs/remotes/origin/pr/* 30 | triggers: 31 | - github-pull-request: 32 | github-hooks: true 33 | org-list: 34 | - elastic 35 | allow-whitelist-orgs-as-admins: true 36 | cancel-builds-on-update: true 37 | status-context: devops-ci 38 | -------------------------------------------------------------------------------- /.ci/jobs/elastic+ansible-elasticsearch+pull-request-6x.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: elastic+ansible-elasticsearch+pull-request-6x 4 | display-name: elastic / ansible-elasticsearch - pull-request - 6.x 5 | description: Pull request testing with test kitchen for Elasticsearch 6.x 6 | parameters: 7 | - string: 8 | name: VERSION 9 | default: 6.x 10 | description: Elasticsearch major version 11 | scm: 12 | - git: 13 | branches: 14 | - $ghprbActualCommit 15 | refspec: +refs/pull/*:refs/remotes/origin/pr/* 16 | axes: 17 | - axis: 18 | type: slave 19 | name: label 20 | values: 21 | - linux 22 | - axis: 23 | name: OS 24 | filename: elasticsearch/test/matrix-6x.yml 25 | type: yaml 26 | - axis: 27 | name: TEST_TYPE 28 | filename: elasticsearch/test/matrix-6x.yml 29 | type: yaml 30 | triggers: 31 | - github-pull-request: 32 | github-hooks: true 33 | org-list: 34 | - elastic 35 | allow-whitelist-orgs-as-admins: true 36 | cancel-builds-on-update: true 37 | status-context: devops-ci 38 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | **Describe the feature**: 22 | 23 | 24 | 25 | **Elasticsearch version** 26 | 27 | **Role version**: (If using main please specify github sha) 28 | 29 | **JVM version** (`java -version`): 30 | 31 | **OS version** (`uname -a` if on a Unix-like system): 32 | 33 | **Description of the problem including expected versus actual behaviour**: 34 | 35 | **Playbook**: 36 | Please specify the full playbook used to reproduce this issue. 37 | 38 | **Provide logs from Ansible**: 39 | 40 | **ES Logs if relevant**: 41 | -------------------------------------------------------------------------------- /.ci/jobs/elastic+ansible-elasticsearch+main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: elastic+ansible-elasticsearch+main 4 | display-name: elastic / ansible-elasticsearch - main 5 | description: Main branch testing with test kitchen 6 | parameters: 7 | - string: 8 | name: branch_specifier 9 | default: main 10 | description: the Git branch specifier to build (<branchName>, <tagName>, 11 | <commitId>, etc.) 12 | - string: 13 | name: VERSION 14 | default: 7.x 15 | description: Elasticsearch major version 16 | axes: 17 | - axis: 18 | type: slave 19 | name: label 20 | values: 21 | - linux 22 | - axis: 23 | name: OS 24 | filename: elasticsearch/test/matrix.yml 25 | type: yaml 26 | - axis: 27 | name: TEST_TYPE 28 | filename: elasticsearch/test/matrix.yml 29 | type: yaml 30 | triggers: 31 | - timed: H H(02-04) * * * 32 | publishers: 33 | - slack: 34 | notify-back-to-normal: True 35 | notify-every-failure: True 36 | room: infra-release-notify 37 | team-domain: elastic 38 | auth-token-id: release-slack-integration-token 39 | auth-token-credential-id: release-slack-integration-token 40 | -------------------------------------------------------------------------------- /.ci/jobs/elastic+ansible-elasticsearch+main-6x.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: elastic+ansible-elasticsearch+main-6x 4 | display-name: elastic / ansible-elasticsearch - main - 6.x 5 | description: Main branch testing with test kitchen for Elasticsearch 6.x 6 | parameters: 7 | - string: 8 | name: branch_specifier 9 | default: main 10 | description: the Git branch specifier to build (<branchName>, <tagName>, 11 | <commitId>, etc.) 12 | - string: 13 | name: VERSION 14 | default: 6.x 15 | description: Elasticsearch major version 16 | axes: 17 | - axis: 18 | type: slave 19 | name: label 20 | values: 21 | - linux 22 | - axis: 23 | name: OS 24 | filename: elasticsearch/test/matrix-6x.yml 25 | type: yaml 26 | - axis: 27 | name: TEST_TYPE 28 | filename: elasticsearch/test/matrix-6x.yml 29 | type: yaml 30 | triggers: 31 | - timed: H H(02-04) * * * 32 | publishers: 33 | - slack: 34 | notify-back-to-normal: True 35 | notify-every-failure: True 36 | room: infra-release-notify 37 | team-domain: elastic 38 | auth-token-id: release-slack-integration-token 39 | auth-token-credential-id: release-slack-integration-token 40 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Number of days of inactivity before an issue becomes stale 3 | daysUntilStale: 90 4 | 5 | # Number of days of inactivity before an stale issue is closed 6 | daysUntilClose: 30 7 | 8 | # Label to use when marking an issue as stale 9 | staleLabel: triage/stale 10 | 11 | issues: 12 | # Comment to post when marking an issue as stale. 13 | markComment: > 14 | This issue has been automatically marked as stale because it has not had 15 | recent activity. It will be closed if no further activity occurs. Thank you 16 | for your contributions. 17 | # Comment to post when closing a stale issue. 18 | closeComment: > 19 | This issue has been automatically closed because it has not had recent 20 | activity since being marked as stale. 21 | pulls: 22 | # Comment to post when marking a PR as stale. 23 | markComment: > 24 | This PR has been automatically marked as stale because it has not had 25 | recent activity. It will be closed if no further activity occurs. Thank you 26 | for your contributions. 27 | 28 | To track this PR (even if closed), please open a corresponding issue if one 29 | does not already exist. 30 | # Comment to post when closing a stale PR. 31 | closeComment: > 32 | This PR has been automatically closed because it has not had recent 33 | activity since being marked as stale. 34 | 35 | Please reopen when work resumes. 36 | -------------------------------------------------------------------------------- /tasks/elasticsearch-RedHat-version-lock.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: RedHat - install yum-version-lock 3 | become: yes 4 | yum: 5 | name: yum-plugin-versionlock 6 | state: present 7 | update_cache: yes 8 | 9 | - name: RedHat - check if requested elasticsearch version lock exists 10 | become: yes 11 | shell: 'yum versionlock list | grep {{es_package_name}} | grep -c "{{es_version}}"' 12 | register: es_requested_version_locked 13 | args: 14 | warn: false 15 | failed_when: False 16 | changed_when: False 17 | check_mode: False 18 | 19 | - name: RedHat - lock elasticsearch version 20 | become: yes 21 | shell: yum versionlock delete 0:elasticsearch* ; yum versionlock add {{ es_package_name }}-{{ es_version }} 22 | args: 23 | warn: false 24 | when: 25 | - es_version_lock 26 | - es_requested_version_locked is defined 27 | - es_requested_version_locked.stdout|int == 0 28 | 29 | - name: RedHat - check if any elasticsearch version lock exists 30 | become: yes 31 | shell: yum versionlock list | grep -c elasticsearch 32 | register: es_version_locked 33 | args: 34 | warn: false 35 | failed_when: False 36 | changed_when: False 37 | check_mode: False 38 | 39 | - name: RedHat - unlock elasticsearch version 40 | become: yes 41 | shell: yum versionlock delete 0:elasticsearch* 42 | args: 43 | warn: false 44 | when: 45 | - not es_version_lock 46 | - es_version_locked is defined 47 | - es_version_locked.stdout|int > 0 48 | -------------------------------------------------------------------------------- /tasks/java.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: set fact java_state to present 4 | set_fact: java_state="present" 5 | 6 | - name: set fact java_state to latest 7 | set_fact: java_state="latest" 8 | when: update_java == true 9 | 10 | - name: RedHat - Ensure Java is installed 11 | become: yes 12 | yum: name={{ java }} state={{java_state}} 13 | when: ansible_os_family == 'RedHat' 14 | 15 | - name: Get the installed java path 16 | shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0-openjdk | head -1" 17 | become: yes 18 | register: java_full_path 19 | failed_when: False 20 | changed_when: False 21 | check_mode: no 22 | when: ansible_os_family == 'RedHat' 23 | 24 | - name: correct java version selected 25 | alternatives: 26 | name: java 27 | path: "{{ java_full_path.stdout }}" 28 | link: /usr/bin/java 29 | when: ansible_os_family == 'RedHat' and java_full_path is defined 30 | 31 | - name: Refresh java repo 32 | become: yes 33 | apt: update_cache=yes 34 | changed_when: false 35 | when: ansible_os_family == 'Debian' 36 | 37 | - name: Debian - Ensure Java is installed 38 | become: yes 39 | apt: name={{ java }} state={{java_state}} 40 | when: ansible_os_family == 'Debian' 41 | 42 | - name: register open_jdk version 43 | shell: java -version 2>&1 | grep OpenJDK 44 | register: open_jdk 45 | ignore_errors: yes 46 | changed_when: false 47 | check_mode: no 48 | 49 | #https://github.com/docker-library/openjdk/issues/19 - ensures tests pass due to java 8 broken certs 50 | - name: refresh the java ca-certificates 51 | become: yes 52 | command: /var/lib/dpkg/info/ca-certificates-java.postinst configure 53 | when: ansible_distribution == 'Ubuntu' and open_jdk.rc == 0 54 | changed_when: false 55 | -------------------------------------------------------------------------------- /test/integration/security.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Elasticsearch Xpack HTTP different keystore and truststore with password 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_config_6x: 10 | xpack.security.authc.realms.file1.order: 0 11 | xpack.security.authc.realms.file1.type: file 12 | es_config_7x: 13 | xpack.security.authc.realms.file.file1.order: 0 14 | es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" 15 | es_api_basic_auth_username: elastic 16 | es_api_basic_auth_password: changeme 17 | es_api_sleep: 5 18 | es_enable_http_ssl: true 19 | es_enable_transport_ssl: true 20 | es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" 21 | es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" 22 | es_ssl_keystore_password: password1 23 | es_ssl_truststore_password: password2 24 | es_validate_certs: no 25 | es_users: 26 | file: 27 | es_admin: 28 | password: changeMe 29 | roles: 30 | - admin 31 | testUser: 32 | password: changeMeAlso! 33 | roles: 34 | - power_user 35 | - user 36 | es_roles: 37 | file: 38 | admin: 39 | cluster: 40 | - all 41 | indices: 42 | - names: '*' 43 | privileges: 44 | - all 45 | power_user: 46 | cluster: 47 | - monitor 48 | indices: 49 | - names: '*' 50 | privileges: 51 | - all 52 | user: 53 | indices: 54 | - names: '*' 55 | privileges: 56 | - read 57 | -------------------------------------------------------------------------------- /test/integration/license.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Elasticsearch Xpack HTTP different keystore and truststore with password 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_config_6x: 10 | xpack.security.authc.realms.file1.order: 0 11 | xpack.security.authc.realms.file1.type: file 12 | es_config_7x: 13 | xpack.security.authc.realms.file.file1.order: 0 14 | es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" 15 | es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" 16 | es_api_basic_auth_username: elastic 17 | es_api_basic_auth_password: changeme 18 | es_api_sleep: 5 19 | es_enable_http_ssl: true 20 | es_enable_transport_ssl: true 21 | es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" 22 | es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" 23 | es_ssl_keystore_password: password1 24 | es_ssl_truststore_password: password2 25 | es_validate_certs: no 26 | es_users: 27 | file: 28 | es_admin: 29 | password: changeMe 30 | roles: 31 | - admin 32 | testUser: 33 | password: changeMeAlso! 34 | roles: 35 | - power_user 36 | - user 37 | es_roles: 38 | file: 39 | admin: 40 | cluster: 41 | - all 42 | indices: 43 | - names: '*' 44 | privileges: 45 | - all 46 | power_user: 47 | cluster: 48 | - monitor 49 | indices: 50 | - names: '*' 51 | privileges: 52 | - all 53 | user: 54 | indices: 55 | - names: '*' 56 | privileges: 57 | - read 58 | -------------------------------------------------------------------------------- /tasks/compatibility-variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # It is possible to set these are defaults with messy jinja templating one liners however: 4 | # 1. That is really hard to read and debug 5 | # 2. When running multiple plays with the same role the defaults are not re-evaluated. An example of this 6 | # can be seen in our the https://github.com/elastic/ansible-elasticsearch/blob/main/test/integration/xpack.yml 7 | # integration test and in the Multi Node server documentation examples https://github.com/elastic/ansible-elasticsearch/blob/main/test/integration/xpack.yml 8 | - name: Set backward compatibility for deprecated es_enable_xpack variable 9 | when: es_enable_xpack is defined and not es_enable_xpack 10 | block: 11 | - name: Set fact oss_version when using es_enable_xpack 12 | set_fact: oss_version=true 13 | - name: Warn about deprecated es_enable_xpack variable 14 | debug: 15 | msg: "WARNING: es_enable_xpack variable is now deprecated. You should use oss_version instead" 16 | 17 | - name: Set the defaults here otherwise they can't be overriden in the same play if the role is called twice 18 | set_fact: 19 | es_repo_name: "{{ es_major_version }}" 20 | es_package_name: "elasticsearch" 21 | es_other_package_name: "elasticsearch-oss" 22 | es_other_repo_name: "{{ 'oss-' + es_major_version }}" 23 | es_other_apt_url: "deb {{ es_repo_base }}/packages/{{ 'oss-' + es_major_version }}/apt stable main" 24 | 25 | - name: Use the oss repo and package 26 | set_fact: 27 | es_repo_name: "{{ 'oss-' + es_major_version }}" 28 | es_other_repo_name: "{{ es_major_version }}" 29 | es_other_apt_url: "deb {{ es_repo_base }}/packages/{{ es_major_version }}/apt stable main" 30 | es_package_name: "elasticsearch-oss" 31 | es_other_package_name: "elasticsearch" 32 | when: 33 | - oss_version 34 | 35 | - name: Set the URL scheme to https if SSL/TLS is enabled 36 | set_fact: 37 | es_api_scheme: "https" 38 | when: es_enable_http_ssl | bool 39 | -------------------------------------------------------------------------------- /.ci/jobs/defaults.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ##### GLOBAL METADATA 4 | 5 | - meta: 6 | cluster: devops-ci 7 | 8 | ##### JOB DEFAULTS 9 | 10 | - job: 11 | project-type: matrix 12 | logrotate: 13 | daysToKeep: 30 14 | numToKeep: 100 15 | properties: 16 | - github: 17 | url: https://github.com/elastic/ansible-elasticsearch/ 18 | - inject: 19 | properties-content: HOME=$JENKINS_HOME 20 | concurrent: true 21 | node: master 22 | scm: 23 | - git: 24 | name: origin 25 | credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba 26 | reference-repo: /var/lib/jenkins/.git-references/ansible-elasticsearch.git 27 | branches: 28 | - ${branch_specifier} 29 | url: git@github.com:elastic/ansible-elasticsearch.git 30 | basedir: elasticsearch 31 | wipe-workspace: 'False' 32 | vault: 33 | role_id: cff5d4e0-61bf-2497-645f-fcf019d10c13 34 | builders: 35 | - shell: |- 36 | #!/usr/local/bin/runbld 37 | set -euo pipefail 38 | 39 | export RBENV_VERSION='2.5.7' 40 | export PATH="$HOME/.rbenv/bin:$PATH" 41 | eval "$(rbenv init -)" 42 | rbenv local $RBENV_VERSION 43 | 44 | export ES_XPACK_LICENSE_FILE="$(pwd)/license.json" 45 | echo "Getting xpack_license from secrets service" 46 | set +x 47 | VAULT_TOKEN=$( curl -s -X POST -H "Content-Type: application/json" -L -d "{\"role_id\":\"$VAULT_ROLE_ID\",\"secret_id\":\"$VAULT_SECRET_ID\"}" $VAULT_ADDR/v1/auth/approle/login | jq -r '.auth.client_token' ) 48 | curl -s -L -H "X-Vault-Token:$VAULT_TOKEN" $VAULT_ADDR/v1/secret/devops-ci/ansible-elasticsearch/xpack_license | jq -r '.data.value' > ${ES_XPACK_LICENSE_FILE} 49 | set -x 50 | echo "Finished getting xpack_license from secrets service" 51 | make setup 52 | make verify VERSION=$VERSION PATTERN=$TEST_TYPE-$OS 53 | wrappers: 54 | - ansicolor 55 | - timeout: 56 | type: absolute 57 | timeout: 360 58 | fail: true 59 | - timestamps 60 | -------------------------------------------------------------------------------- /tasks/snapshot-release.yml: -------------------------------------------------------------------------------- 1 | # These tasks are to run ansible-elasticsearch using pre-release snapshot builds 2 | # This should only be used for testing purposes and can be enabled by setting 3 | # es_use_snapshot_release: true 4 | 5 | - name: detect if we need the .deb or .rpm 6 | set_fact: 7 | package_type: "{{ 'deb' if (ansible_os_family == 'Debian') else 'rpm' }}" 8 | 9 | - name: get the minor version 10 | set_fact: 11 | minor_version: "{{ es_version.split('.')[0:2] | join('.')}}" 12 | 13 | - name: set the package_name 14 | set_fact: 15 | package_name: "{{ es_package_name + '-' + es_version + '-SNAPSHOT.' + package_type }}" 16 | 17 | - name: generate the artifacts url 18 | set_fact: 19 | artifacts_url: "{{ 'https://artifacts-api.elastic.co/v1/search/' + minor_version + '/' + package_name }}" 20 | 21 | - name: get latest snapshot build 22 | uri: 23 | url: "{{ artifacts_url }}" 24 | return_contents: true 25 | register: snapshots 26 | retries: 5 27 | delay: 1 28 | ignore_errors: true 29 | until: "'status' in snapshots and snapshots.status == 200" 30 | check_mode: no 31 | 32 | - name: use the custom package url instead of the repository 33 | set_fact: 34 | es_custom_package_url: "{{ snapshots.json['packages'][package_name]['url'] }}" 35 | es_use_repository: false 36 | 37 | - name: set snapshot urls for es_plugins when it is defined 38 | when: es_plugins is defined 39 | block: 40 | - name: split up the snapshot url so we can create the plugin url 41 | set_fact: 42 | split_url: "{{ es_custom_package_url.split('/') }}" 43 | 44 | - name: set base plugin url 45 | set_fact: 46 | plugin_url: "{{ split_url[0] + '//' + split_url[2:5]|join('/') + '/elasticsearch-plugins/'}}" 47 | 48 | - name: create es_plugins with the snapshot url 49 | set_fact: 50 | es_plugins_temp: "{{ es_plugins_temp|default([]) + [{'plugin': item.plugin, 'url': plugin_url + item.plugin + '/' + item.plugin + '-' + es_version + '-SNAPSHOT.zip'}] }}" 51 | with_items: "{{ es_plugins }}" 52 | 53 | - name: override the original es_plugins with the snapshot version 54 | set_fact: 55 | es_plugins: "{{ es_plugins_temp }}" 56 | -------------------------------------------------------------------------------- /filter_plugins/custom.py: -------------------------------------------------------------------------------- 1 | __author__ = "dale mcdiarmid" 2 | 3 | import re 4 | import os.path 5 | from six import string_types 6 | 7 | 8 | def modify_list(values=[], pattern="", replacement="", ignorecase=False): 9 | """ Perform a `re.sub` on every item in the list""" 10 | if ignorecase: 11 | flags = re.I 12 | else: 13 | flags = 0 14 | _re = re.compile(pattern, flags=flags) 15 | return [_re.sub(replacement, value) for value in values] 16 | 17 | 18 | def append_to_list(values=[], suffix=""): 19 | if isinstance(values, string_types): 20 | values = values.split(",") 21 | return [str(value + suffix) for value in values] 22 | 23 | 24 | def array_to_str(values=[], separator=","): 25 | return separator.join(values) 26 | 27 | 28 | def extract_role_users(users={}, exclude_users=[]): 29 | role_users = [] 30 | for user, details in list(users.items()): 31 | if user not in exclude_users and "roles" in details: 32 | for role in details["roles"]: 33 | role_users.append(role + ":" + user) 34 | return role_users 35 | 36 | 37 | def filename(filename=""): 38 | return os.path.splitext(os.path.basename(filename))[0] 39 | 40 | 41 | def remove_reserved(user_roles={}): 42 | not_reserved = [] 43 | for user_role, details in list(user_roles.items()): 44 | if ( 45 | not "metadata" in details 46 | or not "_reserved" in details["metadata"] 47 | or not details["metadata"]["_reserved"] 48 | ): 49 | not_reserved.append(user_role) 50 | return not_reserved 51 | 52 | 53 | def filter_reserved(users_role={}): 54 | reserved = [] 55 | for user_role, details in list(users_role.items()): 56 | if ( 57 | "metadata" in details 58 | and "_reserved" in details["metadata"] 59 | and details["metadata"]["_reserved"] 60 | ): 61 | reserved.append(user_role) 62 | return reserved 63 | 64 | 65 | class FilterModule(object): 66 | def filters(self): 67 | return { 68 | "modify_list": modify_list, 69 | "append_to_list": append_to_list, 70 | "filter_reserved": filter_reserved, 71 | "array_to_str": array_to_str, 72 | "extract_role_users": extract_role_users, 73 | "remove_reserved": remove_reserved, 74 | "filename": filename, 75 | } 76 | -------------------------------------------------------------------------------- /templates/elasticsearch.j2: -------------------------------------------------------------------------------- 1 | ################################ 2 | # Elasticsearch 3 | ################################ 4 | 5 | # Elasticsearch home directory 6 | ES_HOME={{es_home}} 7 | 8 | # Elasticsearch temp directory 9 | ES_TMPDIR={{ es_tmp_dir }} 10 | 11 | # Elasticsearch Java path 12 | {% if es_java_home | length > 0 %} 13 | ES_JAVA_HOME={{ es_java_home }} 14 | {% else %} 15 | #ES_JAVA_HOME= 16 | {% endif %} 17 | 18 | # Elasticsearch configuration directory 19 | ES_PATH_CONF={{ es_conf_dir }} 20 | 21 | # Elasticsearch PID directory 22 | PID_DIR={{ es_pid_dir }} 23 | 24 | # Additional Java OPTS 25 | ES_JAVA_OPTS="{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}" 26 | 27 | # Configure restart on package upgrade (true, every other setting will lead to not restarting) 28 | #RESTART_ON_UPGRADE=true 29 | 30 | ################################ 31 | # Elasticsearch service 32 | ################################ 33 | 34 | # SysV init.d 35 | # 36 | # The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process 37 | ES_STARTUP_SLEEP_TIME=5 38 | 39 | ################################ 40 | # System properties 41 | ################################ 42 | 43 | # Specifies the maximum file descriptor number that can be opened by this process 44 | # When using Systemd, this setting is ignored and the LimitNOFILE defined in 45 | # /usr/lib/systemd/system/elasticsearch.service takes precedence 46 | #MAX_OPEN_FILES=65535 47 | {% if es_max_open_files is defined %} 48 | MAX_OPEN_FILES={{es_max_open_files}} 49 | {% endif %} 50 | 51 | # The maximum number of bytes of memory that may be locked into RAM 52 | # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option 53 | # in elasticsearch.yml. 54 | # When using systemd, LimitMEMLOCK must be set in a unit file such as 55 | # /etc/systemd/system/elasticsearch.service.d/override.conf. 56 | #MAX_LOCKED_MEMORY=unlimited 57 | {% if m_lock_enabled %} 58 | MAX_LOCKED_MEMORY=unlimited 59 | {% endif %} 60 | 61 | # Maximum number of VMA (Virtual Memory Areas) a process can own 62 | # When using Systemd, this setting is ignored and the 'vm.max_map_count' 63 | # property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf 64 | #MAX_MAP_COUNT=262144 65 | {% if es_max_map_count is defined %} 66 | MAX_MAP_COUNT={{es_max_map_count}} 67 | {% endif %} 68 | -------------------------------------------------------------------------------- /defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | es_version: "7.17.0" 3 | es_use_snapshot_release: false 4 | oss_version: false 5 | es_package_name: "elasticsearch" 6 | es_version_lock: false 7 | es_use_repository: true 8 | es_add_repository: true 9 | es_templates_fileglob: "" 10 | es_repo_base: "https://artifacts.elastic.co" 11 | es_apt_key: "{{ es_repo_base }}/GPG-KEY-elasticsearch" 12 | es_apt_url: "deb {{ es_repo_base }}/packages/{{ es_repo_name }}/apt stable main" 13 | es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_repo_name }}/debian stable main" 14 | es_start_service: true 15 | es_java_install: "{{ false if (es_version is version('7.0.0', '>=')) else true }}" 16 | update_java: false 17 | es_restart_on_change: true 18 | es_plugins_reinstall: false 19 | es_templates: false 20 | es_user: elasticsearch 21 | es_group: elasticsearch 22 | es_config: {} 23 | es_config_default: "elasticsearch.j2" 24 | es_config_log4j2: "" 25 | es_config_jvm: "jvm.options.j2" 26 | #Need to provide default directories 27 | es_conf_dir: "/etc/elasticsearch" 28 | es_pid_dir: "/var/run/elasticsearch" 29 | es_tmp_dir: "/tmp" 30 | es_data_dirs: 31 | - "/var/lib/elasticsearch" 32 | es_log_dir: "/var/log/elasticsearch" 33 | es_action_auto_create_index: true 34 | es_max_open_files: 65536 35 | es_max_map_count: 262144 36 | es_allow_downgrades: false 37 | es_xpack_trial: false 38 | #These are used for internal operations performed by ansible. 39 | #They do not affect the current configuration 40 | es_api_scheme: "http" 41 | es_api_host: "localhost" 42 | es_api_port: 9200 43 | es_api_uri: "{{ es_api_scheme }}://{{ es_api_host }}:{{ es_api_port }}" 44 | es_security_api: "{{ '_security' if es_version is version('7.0.0', '>=') else '_xpack/security' }}" 45 | es_license_api: "{{ '_license' if es_version is version('7.0.0', '>=') else '_xpack/license' }}" 46 | es_api_sleep: 15 47 | es_debian_startup_timeout: 10 48 | 49 | # JVM custom parameters 50 | es_java_home: '' 51 | es_jvm_custom_parameters: '' 52 | es_heap_dump_path: "/var/lib/elasticsearch" 53 | 54 | # SSL/TLS parameters 55 | es_enable_auto_ssl_configuration: true 56 | es_enable_http_ssl: false 57 | es_enable_transport_ssl: false 58 | es_ssl_upload: true 59 | es_ssl_keystore: "" 60 | es_ssl_keystore_password: "" 61 | es_ssl_truststore: "" 62 | es_ssl_truststore_password: "" 63 | es_ssl_key: "" 64 | es_ssl_key_password: "" 65 | es_ssl_certificate: "" 66 | es_ssl_certificate_authority: "" 67 | es_ssl_certificate_path: "{{ es_conf_dir }}/certs" 68 | es_ssl_verification_mode: "certificate" 69 | es_validate_certs: "yes" 70 | es_delete_unmanaged_file: true 71 | es_delete_unmanaged_native: true 72 | -------------------------------------------------------------------------------- /tasks/elasticsearch-RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set fact allow_downgrade to no 3 | set_fact: allow_downgrade=no 4 | 5 | - name: set fact allow_downgrade to yes 6 | set_fact: allow_downgrade=yes 7 | when: es_allow_downgrades 8 | 9 | - name: Ensure libselinux-python on CentOS 6.x 10 | become: yes 11 | yum: name=libselinux-python state=present update_cache=yes 12 | when: ( ansible_distribution == "CentOS" ) and ( ansible_distribution_major_version == "6" ) 13 | 14 | - name: RedHat - add Elasticsearch repo 15 | become: yes 16 | template: 17 | src: 'elasticsearch.repo' 18 | dest: '/etc/yum.repos.d/elasticsearch-{{ es_repo_name }}.repo' 19 | when: es_use_repository and es_add_repository 20 | 21 | - name: RedHat - remove unused Elasticsearch repo 22 | become: yes 23 | file: 24 | path: '/etc/yum.repos.d/elasticsearch-{{ es_other_repo_name }}.repo' 25 | state: absent 26 | when: es_use_repository 27 | 28 | - name: RedHat - include versionlock 29 | include: elasticsearch-RedHat-version-lock.yml 30 | 31 | - name: RedHat - Remove the other elasticsearch package if switching between OSS and standard 32 | become: yes 33 | yum: 34 | name: '{{ es_other_package_name }}' 35 | state: 'absent' 36 | 37 | - name: Include optional user and group creation. 38 | when: (es_user_id is defined) and (es_group_id is defined) 39 | include: elasticsearch-optional-user.yml 40 | 41 | - name: RedHat - Install Elasticsearch 42 | become: yes 43 | yum: 44 | name: '{{ es_package_name }}{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %}' 45 | state: present 46 | update_cache: yes 47 | allow_downgrade: '{{ allow_downgrade }}' 48 | when: es_use_repository and not ansible_check_mode 49 | register: redhat_elasticsearch_install_from_repo 50 | notify: restart elasticsearch 51 | until: redhat_elasticsearch_install_from_repo.rc == 0 52 | retries: 5 53 | delay: 10 54 | environment: 55 | ES_PATH_CONF: "{{ es_conf_dir }}" 56 | 57 | # workaround due to https://github.com/ansible/ansible/issues/66977 58 | - set_fact: 59 | es_rpm_url: "{% if es_version is version('7.0.0', '>=') %}{{ es_package_url }}-{{ es_version }}-x86_64.rpm{% else %}{{ es_package_url }}-{{ es_version }}.rpm{% endif %}" 60 | 61 | - name: RedHat - Install Elasticsearch from url 62 | become: yes 63 | yum: 64 | name: '{% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_rpm_url }}{% endif %}' 65 | state: present 66 | when: not es_use_repository 67 | register: elasticsearch_install_from_package 68 | notify: restart elasticsearch 69 | -------------------------------------------------------------------------------- /tasks/elasticsearch-config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure Elasticsearch Node 3 | 4 | #Create conf directory 5 | - name: Create Configuration Directory 6 | become: yes 7 | file: 8 | path: "{{ es_conf_dir }}" 9 | state: directory 10 | owner: root 11 | group: "{{ es_group }}" 12 | mode: "2750" 13 | 14 | #Create pid directory 15 | - name: Create PID Directory 16 | become: yes 17 | file: 18 | path: "{{ es_pid_dir }}" 19 | state: directory 20 | owner: "{{ es_user }}" 21 | group: "{{ es_group }}" 22 | mode: "755" 23 | 24 | #Create required directories 25 | - name: Create Others Directories 26 | become: yes 27 | file: 28 | path: "{{ item }}" 29 | state: directory 30 | owner: "{{ es_user }}" 31 | group: "{{ es_group }}" 32 | mode: "2750" 33 | with_items: 34 | - "{{ es_log_dir }}" 35 | - "{{ es_data_dirs }}" 36 | 37 | #Copy the config template 38 | - name: Copy Configuration File 39 | become: yes 40 | template: 41 | src: elasticsearch.yml.j2 42 | dest: "{{ es_conf_dir }}/elasticsearch.yml" 43 | owner: root 44 | group: "{{ es_group }}" 45 | mode: "660" 46 | force: yes 47 | register: system_change 48 | notify: restart elasticsearch 49 | 50 | #Copy the default file 51 | - name: Copy Default File 52 | become: yes 53 | template: 54 | src: "{{ es_config_default }}" 55 | dest: "{{ default_file }}" 56 | owner: root 57 | group: "{{ es_group }}" 58 | mode: "660" 59 | force: yes 60 | notify: restart elasticsearch 61 | 62 | #Copy the systemd specific file if systemd is installed 63 | - when: use_system_d and (m_lock_enabled or es_max_open_files is defined) 64 | become: yes 65 | block: 66 | - name: Make sure destination dir exists 67 | file: 68 | path: "{{ sysd_config_file | dirname }}" 69 | state: directory 70 | mode: "755" 71 | 72 | - name: Copy specific ElasticSearch Systemd config file 73 | template: 74 | src: "override.conf.j2" 75 | dest: "{{ sysd_config_file }}" 76 | owner: root 77 | group: "{{ es_group }}" 78 | mode: "644" 79 | force: yes 80 | notify: 81 | - reload systemd configuration 82 | - restart elasticsearch 83 | 84 | - name: Copy jvm.options File 85 | become: yes 86 | template: 87 | src: "{{ es_config_jvm }}" 88 | dest: "{{ es_conf_dir }}/jvm.options" 89 | owner: root 90 | group: "{{ es_group }}" 91 | mode: "660" 92 | force: yes 93 | notify: restart elasticsearch 94 | 95 | - name: Copy log4j2.properties File 96 | become: yes 97 | template: 98 | src: "{{ es_config_log4j2 }}" 99 | dest: "{{ es_conf_dir }}/log4j2.properties" 100 | owner: root 101 | group: "{{ es_group }}" 102 | mode: "660" 103 | force: yes 104 | notify: restart elasticsearch 105 | when: es_config_log4j2 != '' 106 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | bcrypt_pbkdf (1.0.1) 5 | builder (3.2.4) 6 | ed25519 (1.2.4) 7 | equatable (0.5.0) 8 | erubi (1.9.0) 9 | ffi (1.12.1) 10 | gssapi (1.3.0) 11 | ffi (>= 1.0.1) 12 | gyoku (1.3.1) 13 | builder (>= 2.1.2) 14 | httpclient (2.8.3) 15 | kitchen-ansible (0.50.0) 16 | net-ssh (>= 3) 17 | test-kitchen (>= 1.4) 18 | kitchen-docker (2.9.0) 19 | test-kitchen (>= 1.0.0) 20 | license-acceptance (1.0.11) 21 | pastel (~> 0.7) 22 | tomlrb (~> 1.2) 23 | tty-box (~> 0.3) 24 | tty-prompt (~> 0.18) 25 | little-plugger (1.1.4) 26 | logging (2.2.2) 27 | little-plugger (~> 1.1) 28 | multi_json (~> 1.10) 29 | mixlib-install (3.11.18) 30 | mixlib-shellout 31 | mixlib-versioning 32 | thor 33 | mixlib-shellout (2.4.4) 34 | mixlib-versioning (1.2.7) 35 | multi_json (1.14.1) 36 | necromancer (0.4.0) 37 | net-scp (2.0.0) 38 | net-ssh (>= 2.6.5, < 6.0.0) 39 | net-ssh (5.2.0) 40 | net-ssh-gateway (2.0.0) 41 | net-ssh (>= 4.0.0) 42 | nori (2.6.0) 43 | pastel (0.7.2) 44 | equatable (~> 0.5.0) 45 | tty-color (~> 0.4.0) 46 | rubyntlm (0.6.2) 47 | rubyzip (2.0.0) 48 | strings (0.1.5) 49 | strings-ansi (~> 0.1) 50 | unicode-display_width (~> 1.5) 51 | unicode_utils (~> 1.4) 52 | strings-ansi (0.1.0) 53 | test-kitchen (2.2.5) 54 | bcrypt_pbkdf (~> 1.0) 55 | ed25519 (~> 1.2) 56 | license-acceptance (~> 1.0, >= 1.0.11) 57 | mixlib-install (~> 3.6) 58 | mixlib-shellout (>= 1.2, < 3.0) 59 | net-scp (>= 1.1, < 3.0) 60 | net-ssh (>= 2.9, < 6.0) 61 | net-ssh-gateway (>= 1.2, < 3.0) 62 | thor (~> 0.19) 63 | winrm (~> 2.0) 64 | winrm-elevated (~> 1.0) 65 | winrm-fs (~> 1.1) 66 | thor (0.20.3) 67 | timers (4.3.0) 68 | tomlrb (1.2.8) 69 | tty-box (0.3.0) 70 | pastel (~> 0.7.2) 71 | strings (~> 0.1.4) 72 | tty-cursor (~> 0.6.0) 73 | tty-color (0.4.3) 74 | tty-cursor (0.6.1) 75 | tty-prompt (0.18.1) 76 | necromancer (~> 0.4.0) 77 | pastel (~> 0.7.0) 78 | timers (~> 4.0) 79 | tty-cursor (~> 0.6.0) 80 | tty-reader (~> 0.5.0) 81 | tty-reader (0.5.0) 82 | tty-cursor (~> 0.6.0) 83 | tty-screen (~> 0.6.4) 84 | wisper (~> 2.0.0) 85 | tty-screen (0.6.5) 86 | unicode-display_width (1.6.0) 87 | unicode_utils (1.4.0) 88 | winrm (2.3.4) 89 | builder (>= 2.1.2) 90 | erubi (~> 1.8) 91 | gssapi (~> 1.2) 92 | gyoku (~> 1.0) 93 | httpclient (~> 2.2, >= 2.2.0.2) 94 | logging (>= 1.6.1, < 3.0) 95 | nori (~> 2.0) 96 | rubyntlm (~> 0.6.0, >= 0.6.1) 97 | winrm-elevated (1.1.1) 98 | winrm (~> 2.0) 99 | winrm-fs (~> 1.0) 100 | winrm-fs (1.3.4) 101 | erubi (~> 1.8) 102 | logging (>= 1.6.1, < 3.0) 103 | rubyzip (~> 2.0) 104 | winrm (~> 2.0) 105 | wisper (2.0.0) 106 | 107 | PLATFORMS 108 | ruby 109 | 110 | DEPENDENCIES 111 | kitchen-ansible 112 | kitchen-docker 113 | net-ssh 114 | test-kitchen 115 | 116 | BUNDLED WITH 117 | 1.17.0 118 | -------------------------------------------------------------------------------- /test/integration/trial.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Setup Elasticsearch with security enabled and a trial license 3 | hosts: localhost 4 | post_tasks: 5 | - include: elasticsearch/test/integration/debug.yml 6 | roles: 7 | - elasticsearch 8 | vars: 9 | es_config_6x: 10 | xpack.security.authc.realms.file1.order: 0 11 | xpack.security.authc.realms.file1.type: file 12 | xpack.security.authc.realms.native1.order: 1 13 | xpack.security.authc.realms.native1.type: native 14 | es_config_7x: 15 | xpack.security.authc.realms.file.file1.order: 0 16 | xpack.security.authc.realms.native.native1.order: 1 17 | es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" 18 | es_xpack_trial: true 19 | es_api_basic_auth_username: elastic 20 | es_api_basic_auth_password: changeme 21 | es_api_sleep: 5 22 | es_enable_http_ssl: false 23 | es_enable_transport_ssl: true 24 | es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" 25 | es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" 26 | es_ssl_keystore_password: password1 27 | es_ssl_truststore_password: password2 28 | es_validate_certs: no 29 | es_role_mapping: 30 | power_user: 31 | - "cn=admins,dc=example,dc=com" 32 | user: 33 | - "cn=users,dc=example,dc=com" 34 | - "cn=admins,dc=example,dc=com" 35 | es_users: 36 | native: 37 | kibana4_server: 38 | password: changeMe 39 | roles: 40 | - kibana4_server 41 | logstash_system: 42 | #this should be successfully modified 43 | password: aNewLogstashPassword 44 | #this will be ignored 45 | roles: 46 | - kibana4_server 47 | elastic: 48 | password: elasticChanged 49 | file: 50 | es_admin: 51 | password: changeMe 52 | roles: 53 | - admin 54 | testUser: 55 | password: changeMeAlso! 56 | roles: 57 | - power_user 58 | - user 59 | es_roles: 60 | file: 61 | admin: 62 | cluster: 63 | - all 64 | indices: 65 | - names: '*' 66 | privileges: 67 | - all 68 | power_user: 69 | cluster: 70 | - monitor 71 | indices: 72 | - names: '*' 73 | privileges: 74 | - all 75 | user: 76 | indices: 77 | - names: '*' 78 | privileges: 79 | - read 80 | kibana4_server: 81 | cluster: 82 | - monitor 83 | indices: 84 | - names: '.kibana' 85 | privileges: 86 | - all 87 | native: 88 | logstash: 89 | cluster: 90 | - manage_index_templates 91 | indices: 92 | - names: 'logstash-*' 93 | privileges: 94 | - write 95 | - delete 96 | - create_index 97 | #this will be ignored - its reserved 98 | logstash_system: 99 | cluster: 100 | - manage_index_templates 101 | indices: 102 | - names: 'logstash-*' 103 | privileges: 104 | - write 105 | - delete 106 | - create_index 107 | -------------------------------------------------------------------------------- /tasks/xpack/security/elasticsearch-security.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #Security specific configuration done here 3 | 4 | #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 5 | 6 | #-----------------------------Create Bootstrap User----------------------------------- 7 | ### START BLOCK elasticsearch keystore ### 8 | - name: create the elasticsearch keystore 9 | block: 10 | - name: create the keystore if it doesn't exist yet 11 | become: yes 12 | command: > 13 | {{es_home}}/bin/elasticsearch-keystore create 14 | args: 15 | creates: "{{ es_conf_dir }}/elasticsearch.keystore" 16 | environment: 17 | ES_PATH_CONF: "{{ es_conf_dir }}" 18 | 19 | - name: Check if bootstrap password is set 20 | become: yes 21 | command: > 22 | {{es_home}}/bin/elasticsearch-keystore list 23 | register: list_keystore 24 | changed_when: False 25 | environment: 26 | ES_PATH_CONF: "{{ es_conf_dir }}" 27 | check_mode: no 28 | 29 | - name: Create Bootstrap password for elastic user 30 | become: yes 31 | shell: echo {{ es_api_basic_auth_password | quote }} | {{ es_home }}/bin/elasticsearch-keystore add -x 'bootstrap.password' 32 | when: 33 | - es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines 34 | environment: 35 | ES_PATH_CONF: "{{ es_conf_dir }}" 36 | no_log: true 37 | 38 | - name: Remove keystore entries 39 | become: yes 40 | command: > 41 | {{ es_home }}/bin/elasticsearch-keystore remove '{{ item.key }}' 42 | with_items: "{{ es_keystore_entries }}" 43 | when: 44 | - es_keystore_entries is defined and es_keystore_entries | length > 0 45 | - item.state is defined and item.state == 'absent' 46 | - item.key in list_keystore.stdout_lines 47 | - ('bootstrap.password' not in item.key) 48 | no_log: true 49 | 50 | - name: Reload keystore entries 51 | become: yes 52 | command: > 53 | {{es_home}}/bin/elasticsearch-keystore list 54 | register: list_keystore 55 | changed_when: False 56 | environment: 57 | ES_PATH_CONF: "{{ es_conf_dir }}" 58 | check_mode: no 59 | 60 | - name: Add keystore entries 61 | become: yes 62 | shell: echo {{ item.value | quote }} | {{ es_home }}/bin/elasticsearch-keystore add -x -f {{ item.key }} 63 | with_items: "{{ es_keystore_entries }}" 64 | when: 65 | - es_keystore_entries is defined and es_keystore_entries | length > 0 66 | - item.state is undefined or item.state == 'present' 67 | - item.force|default(False) or ( not item.force|default(False) and item.key not in list_keystore.stdout_lines ) 68 | - ('bootstrap.password' not in item.key) 69 | no_log: true 70 | 71 | 72 | ### END BLOCK elasticsearch keystore ### 73 | 74 | #-----------------------------FILE BASED REALM---------------------------------------- 75 | 76 | - include: elasticsearch-security-file.yml 77 | when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) 78 | 79 | #-----------------------------ROLE MAPPING ---------------------------------------- 80 | 81 | #Copy Roles files 82 | - name: Copy role_mapping.yml file for instance 83 | become: yes 84 | template: 85 | src: security/role_mapping.yml.j2 86 | dest: "{{ es_conf_dir }}/role_mapping.yml" 87 | owner: root 88 | group: "{{ es_group }}" 89 | mode: "0660" 90 | force: yes 91 | when: es_role_mapping is defined 92 | -------------------------------------------------------------------------------- /templates/elasticsearch.yml.j2: -------------------------------------------------------------------------------- 1 | 2 | {% if es_config %} 3 | {{ es_config | to_nice_yaml(indent=2) }} 4 | {% endif %} 5 | 6 | {% if es_config['cluster.name'] is not defined %} 7 | cluster.name: elasticsearch 8 | {% endif %} 9 | 10 | {% if es_config['node.name'] is not defined %} 11 | node.name: {{inventory_hostname}} 12 | {% endif %} 13 | 14 | #################################### Paths #################################### 15 | 16 | # Path to directory containing configuration (this file and logging.yml): 17 | 18 | path.data: {{ es_data_dirs | array_to_str }} 19 | 20 | path.logs: {{ es_log_dir }} 21 | 22 | {% if es_path_repo is defined %} 23 | path.repo: {{ es_path_repo }} 24 | {% endif %} 25 | 26 | {% if es_action_auto_create_index == true %} 27 | action.auto_create_index: true 28 | {% elif not es_action_auto_create_index %} 29 | action.auto_create_index: false 30 | {% else %} 31 | action.auto_create_index: {{ es_action_auto_create_index }} 32 | {% endif %} 33 | 34 | {% if not oss_version and es_api_basic_auth_username is defined and es_api_basic_auth_password is defined %} 35 | xpack.security.enabled: true 36 | 37 | {% if es_enable_transport_ssl and es_enable_auto_ssl_configuration %} 38 | xpack.security.transport.ssl.enabled: true 39 | xpack.security.transport.ssl.verification_mode: "{{ es_ssl_verification_mode }}" 40 | {% if es_ssl_keystore and es_ssl_truststore %} 41 | xpack.security.transport.ssl.keystore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" 42 | xpack.security.transport.ssl.truststore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" 43 | {% elif es_ssl_key and es_ssl_certificate %} 44 | xpack.security.transport.ssl.key: "{{ es_ssl_certificate_path }}/{{ es_ssl_key | basename }}" 45 | xpack.security.transport.ssl.certificate: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate | basename }}" 46 | {% if es_ssl_certificate_authority %} 47 | xpack.security.transport.ssl.certificate_authorities: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" 48 | {% endif %} 49 | {% endif %} 50 | {% endif %} 51 | 52 | {% if es_enable_http_ssl and es_enable_auto_ssl_configuration %} 53 | xpack.security.http.ssl.enabled: true 54 | {% if es_ssl_keystore and es_ssl_truststore %} 55 | xpack.security.http.ssl.keystore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" 56 | xpack.security.http.ssl.truststore.path: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" 57 | {% elif es_ssl_key and es_ssl_certificate %} 58 | xpack.security.http.ssl.key: "{{ es_ssl_certificate_path }}/{{ es_ssl_key | basename }}" 59 | xpack.security.http.ssl.certificate: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate | basename }}" 60 | {% if es_ssl_certificate_authority %} 61 | xpack.security.http.ssl.certificate_authorities: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" 62 | {% endif %} 63 | {% endif %} 64 | {% endif %} 65 | {% endif %} 66 | 67 | {% if es_mail_config is defined %} 68 | xpack.notification.email: 69 | account: 70 | {{ es_mail_config['account'] }}: 71 | profile: {{ es_mail_config['profile'] }} 72 | email_defaults: 73 | from: {{ es_mail_config['from'] }} 74 | smtp: 75 | auth: {{ es_mail_config['require_auth'] }} 76 | host: {{ es_mail_config['host'] }} 77 | port: {{ es_mail_config['port'] }} 78 | {% if es_mail_config['require_auth'] == true -%} 79 | user: {{ es_mail_config['user'] }} 80 | password: {{ es_mail_config['pass'] }} 81 | {%- endif %} 82 | {% endif %} 83 | -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/security_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'json' 3 | require 'pathname' 4 | vars = JSON.parse(File.read('/tmp/vars.json')) 5 | 6 | es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" 7 | username = vars['es_api_basic_auth_username'] 8 | password = vars['es_api_basic_auth_password'] 9 | es_keystore = Pathname.new(vars['es_ssl_keystore']).basename.to_s 10 | es_truststore = Pathname.new(vars['es_ssl_truststore']).basename.to_s 11 | 12 | if vars['es_major_version'] == '7.x' 13 | es_security_api = "_security" 14 | else 15 | es_security_api = "_xpack/security" 16 | end 17 | 18 | shared_examples 'security::init' do |vars| 19 | #Test users file, users_roles and roles.yml 20 | describe file("/etc/elasticsearch/users_roles") do 21 | it { should be_owned_by 'root' } 22 | it { should contain 'admin:es_admin' } 23 | it { should contain 'power_user:testUser' } 24 | end 25 | 26 | describe file("/etc/elasticsearch/users") do 27 | it { should be_owned_by 'root' } 28 | it { should contain 'testUser:' } 29 | it { should contain 'es_admin:' } 30 | end 31 | 32 | describe 'security roles' do 33 | it 'should list the security roles' do 34 | roles = curl_json("#{es_api_url}/#{es_security_api}/role", username='es_admin', password='changeMeAgain') 35 | expect(roles.key?('superuser')) 36 | end 37 | end 38 | 39 | describe file("/etc/elasticsearch/elasticsearch.yml") do 40 | if vars['es_major_version'] == '7.x' 41 | it { should contain 'security.authc.realms.file.file1.order: 0' } 42 | else 43 | it { should contain 'security.authc.realms.file1.order: 0' } 44 | it { should contain 'security.authc.realms.file1.type: file' } 45 | end 46 | it { should contain 'xpack.security.transport.ssl.enabled: true' } 47 | it { should contain 'xpack.security.http.ssl.enabled: true' } 48 | it { should contain es_keystore } 49 | it { should contain es_truststore } 50 | end 51 | 52 | #Test contents of role_mapping.yml 53 | describe file("/etc/elasticsearch/role_mapping.yml") do 54 | it { should be_owned_by 'root' } 55 | it { should contain 'power_user:' } 56 | it { should contain 'user:' } 57 | end 58 | 59 | #check accounts are correct i.e. we can auth and they have the correct roles 60 | describe 'security users' do 61 | result = curl_json("#{es_api_url}/#{es_security_api}/user", username=username, password=password) 62 | it 'should have the elastic user' do 63 | expect(result['elastic']['username']).to eq('elastic') 64 | expect(result['elastic']['roles']).to eq(['superuser']) 65 | expect(result['elastic']['enabled']).to eq(true) 66 | end 67 | it 'should have the kibana user' do 68 | expect(result['kibana']['username']).to eq('kibana') 69 | expect(result['kibana']['roles']).to eq(['kibana_system']) 70 | expect(result['kibana']['enabled']).to eq(true) 71 | end 72 | it 'should have the logstash user' do 73 | expect(result['logstash_system']['username']).to eq('logstash_system') 74 | expect(result['logstash_system']['roles']).to eq(['logstash_system']) 75 | expect(result['logstash_system']['enabled']).to eq(true) 76 | end 77 | end 78 | 79 | describe 'SSL certificate check' do 80 | certificates = curl_json("#{es_api_url}/_ssl/certificates", username=username, password=password) 81 | it 'should list the keystore file' do 82 | expect(certificates.any? { |cert| cert['path'].include? es_keystore }).to be true 83 | end 84 | it 'should list the truststore file' do 85 | expect(certificates.any? { |cert| cert['path'].include? es_truststore }).to be true 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /tasks/elasticsearch-parameters.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Check for mandatory parameters 3 | 4 | - name: Warn about deprecated es_xpack_features variable 5 | debug: 6 | msg: "WARNING: es_xpack_features variable is now deprecated. All feature are now enabled by default" 7 | when: es_xpack_features is defined and not oss_version 8 | 9 | - name: "fail when oss_version is true with es_version >= 7.11.0" 10 | fail: 11 | msg: > 12 | OSS versions are not available for Elasticsearch >= 7.11.0. 13 | See https://www.elastic.co/blog/licensing-change for more details. 14 | when: oss_version and es_version is version('7.11.0', '>=') 15 | 16 | - name: fail when es_proxy_port is not defined or is blank 17 | fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" 18 | when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') 19 | 20 | # If the user attempts to lock memory they must specify a heap size 21 | - name: fail when heap size is not specified when using memory lock 22 | fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified" 23 | when: es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True and es_heap_size is not defined and not ansible_check_mode 24 | 25 | - name: fail when password is not declared when using security 26 | fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" 27 | when: 28 | - not oss_version 29 | - es_api_basic_auth_username is defined 30 | - es_api_basic_auth_password is not defined 31 | 32 | - name: fail when api credentials are not declared when using tls 33 | fail: msg="Enabling tls requires an es_api_basic_auth_username and es_api_basic_auth_password" 34 | when: 35 | - not oss_version 36 | - es_enable_http_ssl or es_enable_transport_ssl 37 | - es_api_basic_auth_username is not defined or es_api_basic_auth_password is not defined 38 | 39 | - name: fail when ssl enabled without defining a key and certificate 40 | fail: msg="Enabling SSL/TLS (es_enable_http_ssl or es_enable_transport_ssl) requires es_ssl_keystore and es_ssl_truststore or es_ssl_key and es_ssl_certificate to be provided" 41 | when: 42 | - es_enable_http_ssl or es_enable_transport_ssl 43 | - (es_ssl_key == "" or es_ssl_certificate == "") 44 | - (es_ssl_keystore == "" or es_ssl_truststore == "") 45 | 46 | - name: set fact file_reserved_users 47 | set_fact: file_reserved_users={{ es_users.file.keys() | list | intersect (reserved_xpack_users) }} 48 | when: es_users is defined and es_users.file is defined and (es_users.file.keys() | list | length > 0) and (es_users.file.keys() | list | intersect (reserved_xpack_users) | length > 0) 49 | 50 | - name: fail when changing users through file realm 51 | fail: 52 | msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM." 53 | when: file_reserved_users | default([]) | length > 0 54 | 55 | - name: set fact m_lock_enabled 56 | set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }} 57 | 58 | - name: set fact use_system_d 59 | set_fact: use_system_d={{ (ansible_distribution == 'Debian' and ansible_distribution_version is version('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version is version('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('15', '>=')) or (ansible_distribution == 'Amazon' and ansible_distribution_version is version('2', '>=')) }} 60 | -------------------------------------------------------------------------------- /tasks/xpack/security/elasticsearch-security-file.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: manage_file_users=false 3 | 4 | - set_fact: manage_file_users=true 5 | when: es_users is defined and es_users.file is defined and es_users.file.keys() | list | length > 0 6 | 7 | # Users migration from elasticsearch < 6.3 versions 8 | - name: Check if old users file exists 9 | become: yes 10 | stat: 11 | path: '{{ es_conf_dir }}/x-pack/users' 12 | register: old_users_file 13 | check_mode: no 14 | 15 | - name: Copy the old users file from the old deprecated location 16 | become: yes 17 | copy: 18 | remote_src: yes 19 | force: no # only copy it if the new path doesn't exist yet 20 | src: "{{ es_conf_dir }}/x-pack/users" 21 | dest: "{{ es_conf_dir }}/users" 22 | group: "{{ es_group }}" 23 | owner: root 24 | when: old_users_file.stat.exists 25 | # End of users migrations 26 | 27 | #List current users 28 | - name: List Users 29 | become: yes 30 | shell: cat {{ es_conf_dir }}/users | awk -F':' '{print $1}' 31 | register: current_file_users 32 | when: manage_file_users 33 | changed_when: False 34 | check_mode: no 35 | 36 | - name: set fact users_to_remove 37 | set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys() | list) }} 38 | when: manage_file_users and es_delete_unmanaged_file 39 | 40 | #Remove users 41 | - name: Remove Users 42 | become: yes 43 | command: > 44 | {{es_home}}/bin/elasticsearch-users userdel {{item}} 45 | with_items: "{{users_to_remove | default([])}}" 46 | when: manage_file_users 47 | environment: 48 | CONF_DIR: "{{ es_conf_dir }}" 49 | ES_PATH_CONF: "{{ es_conf_dir }}" 50 | ES_HOME: "{{es_home}}" 51 | 52 | - name: set fact users_to_add 53 | set_fact: users_to_add={{ es_users.file.keys() | list | difference (current_file_users.stdout_lines) }} 54 | when: manage_file_users and es_delete_unmanaged_file 55 | 56 | #Add users 57 | - name: Add Users 58 | become: yes 59 | command: > 60 | {{es_home}}/bin/elasticsearch-users useradd {{item}} -p {{es_users.file[item].password}} 61 | with_items: "{{ users_to_add | default([]) }}" 62 | when: manage_file_users 63 | no_log: True 64 | environment: 65 | CONF_DIR: "{{ es_conf_dir }}" 66 | ES_PATH_CONF: "{{ es_conf_dir }}" 67 | ES_HOME: "{{es_home}}" 68 | 69 | #Set passwords for all users declared - Required as the useradd will not change existing user passwords 70 | - name: Set User Passwords 71 | become: yes 72 | command: > 73 | {{es_home}}/bin/elasticsearch-users passwd {{ item }} -p {{es_users.file[item].password}} 74 | with_items: "{{ es_users.file.keys() | list }}" 75 | when: manage_file_users 76 | #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. 77 | changed_when: False 78 | no_log: True 79 | environment: 80 | CONF_DIR: "{{ es_conf_dir }}" 81 | ES_PATH_CONF: "{{ es_conf_dir }}" 82 | ES_HOME: "{{es_home}}" 83 | 84 | - name: set fact users_roles 85 | set_fact: users_roles={{es_users.file | extract_role_users () }} 86 | when: manage_file_users 87 | 88 | #Copy Roles files 89 | - name: Copy roles.yml File for Instance 90 | become: yes 91 | template: 92 | src: security/roles.yml.j2 93 | dest: "{{ es_conf_dir }}/roles.yml" 94 | owner: root 95 | group: "{{ es_group }}" 96 | mode: "0660" 97 | force: yes 98 | when: es_roles is defined and es_roles.file is defined 99 | 100 | #Overwrite users_roles file 101 | - name: Copy User Roles 102 | become: yes 103 | template: 104 | src: security/users_roles.j2 105 | dest: "{{ es_conf_dir }}/users_roles" 106 | owner: root 107 | group: "{{ es_group }}" 108 | mode: "0660" 109 | force: yes 110 | when: manage_file_users and users_roles | length > 0 111 | -------------------------------------------------------------------------------- /tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - set_fact: "es_major_version={{ es_version.split('.')[0] }}.x" 4 | when: 5 | - es_major_version is undefined 6 | tags: 7 | - always 8 | 9 | - name: os-specific vars 10 | include_vars: "{{ansible_os_family}}.yml" 11 | tags: 12 | - always 13 | 14 | - name: set compatibility variables 15 | include: compatibility-variables.yml 16 | tags: 17 | - always 18 | 19 | - name: check-set-parameters 20 | include: elasticsearch-parameters.yml 21 | tags: 22 | - always 23 | 24 | - name: use snapshot release 25 | include: snapshot-release.yml 26 | when: es_use_snapshot_release 27 | 28 | - name: include java.yml 29 | include: java.yml 30 | when: es_java_install 31 | tags: 32 | - java 33 | 34 | - name: include elasticsearch.yml 35 | include: elasticsearch.yml 36 | tags: 37 | - install 38 | 39 | - name: include elasticsearch-config.yml 40 | include: elasticsearch-config.yml 41 | tags: 42 | - config 43 | 44 | - name: include elasticsearch-plugins.yml 45 | include: elasticsearch-plugins.yml 46 | when: es_plugins is defined or es_plugins_reinstall 47 | tags: 48 | - plugins 49 | 50 | #We always execute xpack as we may need to remove features 51 | - name: include xpack/elasticsearch-xpack.yml 52 | include: xpack/elasticsearch-xpack.yml 53 | tags: 54 | - xpack 55 | 56 | - name: include elasticsearch-ssl.yml 57 | include: elasticsearch-ssl.yml 58 | when: es_enable_http_ssl or es_enable_transport_ssl 59 | tags: 60 | - xpack 61 | 62 | - name: flush handlers 63 | meta: flush_handlers 64 | 65 | - name: Make sure elasticsearch is started 66 | become: yes 67 | service: name=elasticsearch state=started enabled=yes 68 | when: es_start_service 69 | 70 | - name: Wait for elasticsearch to startup 71 | wait_for: host={{ es_api_host }} port={{ es_api_port }} delay=5 connect_timeout=1 72 | when: es_restarted is defined and es_restarted.changed and es_start_service 73 | 74 | - name: set fact manage_native_realm to false 75 | set_fact: manage_native_realm=false 76 | 77 | - name: set fact manage_native_realm to true 78 | set_fact: manage_native_realm=true 79 | when: 80 | - es_start_service 81 | - not oss_version 82 | - (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) 83 | 84 | # If playbook runs too fast, Native commands could fail as the Native Realm is not yet up 85 | - name: Wait {{ es_api_sleep }} seconds for the Native Realm to come up 86 | wait_for: 87 | timeout: "{{ es_api_sleep }}" 88 | when: manage_native_realm | bool 89 | 90 | - name: activate-license 91 | include: ./xpack/security/elasticsearch-xpack-activation.yml 92 | when: es_start_service and not oss_version and es_xpack_license is defined and es_xpack_license != '' 93 | run_once: True 94 | 95 | - name: activate-trial 96 | include: ./xpack/security/elasticsearch-xpack-trial-activation.yml 97 | when: es_start_service and not oss_version and es_xpack_trial 98 | run_once: True 99 | 100 | #perform security actions here now elasticsearch is started 101 | - name: include xpack/security/elasticsearch-security-native.yml 102 | include: ./xpack/security/elasticsearch-security-native.yml 103 | when: manage_native_realm | bool 104 | run_once: True 105 | 106 | #Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. 107 | #We also do after the native realm to ensure any changes are applied here first and its denf up. 108 | - name: include elasticsearch-template.yml 109 | include: elasticsearch-template.yml 110 | when: es_templates | bool 111 | tags: 112 | - templates 113 | -------------------------------------------------------------------------------- /tasks/elasticsearch-plugins.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed 4 | # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. 5 | - name: set fact es_plugins_reinstall to true 6 | set_fact: es_plugins_reinstall=true 7 | when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none 8 | 9 | - name: set fact list_command 10 | set_fact: list_command="" 11 | #If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent. 12 | - name: set fact list_command check for x-pack 13 | set_fact: list_command="| grep -vE 'x-pack'" 14 | when: not es_plugins_reinstall 15 | 16 | - name: remove x-pack plugin directory when it isn't a plugin 17 | file: 18 | dest: "{{ es_home }}/plugins/x-pack" 19 | state: "absent" 20 | 21 | #List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version. 22 | - name: Check installed elasticsearch plugins 23 | become: yes 24 | shell: "ls {{es_home}}/plugins {{list_command}}" 25 | register: installed_plugins 26 | changed_when: False 27 | ignore_errors: yes 28 | environment: 29 | CONF_DIR: "{{ es_conf_dir }}" 30 | ES_PATH_CONF: "{{ es_conf_dir }}" 31 | ES_INCLUDE: "{{ default_file }}" 32 | check_mode: no 33 | 34 | #if es_plugins_reinstall is set to true we remove ALL plugins 35 | - name: set fact plugins_to_remove to install_plugins.stdout_lines 36 | set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}" 37 | when: es_plugins_reinstall 38 | 39 | #if the plugins listed are different than those requested, we remove those installed but not listed in the config 40 | - name: set fact plugins_to_remove to delete plugins installed but not listed in es_plugins 41 | set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}" 42 | when: not es_plugins_reinstall 43 | 44 | #if es_plugins_reinstall is set to true we (re)install ALL plugins 45 | - name: set fact plugins_to_install to es_plugins 46 | set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | default([]) }}" 47 | when: es_plugins_reinstall 48 | 49 | #if the plugins listed are different than those requested, we install those not installed but listed in the config 50 | - name: set fact to plugins_to_install to those in es_config but not installed 51 | set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | difference(installed_plugins.stdout_lines) | default([]) }}" 52 | when: not es_plugins_reinstall 53 | 54 | # This removes any currently installed plugins (to prevent errors when reinstalling) 55 | - name: Remove elasticsearch plugins 56 | become: yes 57 | command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent" 58 | with_items: "{{ plugins_to_remove | default([]) }}" 59 | notify: restart elasticsearch 60 | register: plugin_removed 61 | environment: 62 | CONF_DIR: "{{ es_conf_dir }}" 63 | ES_PATH_CONF: "{{ es_conf_dir }}" 64 | ES_INCLUDE: "{{ default_file }}" 65 | 66 | - name: Install elasticsearch plugins 67 | become: yes 68 | command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.url | default(item.plugin) }} --batch --silent" 69 | register: plugin_installed 70 | changed_when: plugin_installed.rc|default(0) == 0 71 | with_items: "{{ es_plugins }}" 72 | when: item.plugin in plugins_to_install 73 | notify: restart elasticsearch 74 | environment: 75 | CONF_DIR: "{{ es_conf_dir }}" 76 | ES_PATH_CONF: "{{ es_conf_dir }}" 77 | ES_INCLUDE: "{{ default_file }}" 78 | until: plugin_installed.rc|default(0) == 0 79 | retries: 5 80 | delay: 5 81 | -------------------------------------------------------------------------------- /tasks/elasticsearch-Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: set fact force_install to no 4 | set_fact: force_install=no 5 | 6 | - name: set fact force_install to yes 7 | set_fact: force_install=yes 8 | when: es_allow_downgrades 9 | 10 | - name: Gracefully stop and remove elasticsearch package if switching between OSS and standard 11 | become: yes 12 | block: 13 | - name: Check if the elasticsearch package is installed 14 | shell: "dpkg-query -W -f'${Status}' {{ es_other_package_name }}" 15 | register: elasticsearch_package 16 | failed_when: False 17 | changed_when: False 18 | check_mode: no 19 | 20 | - name: unhold elasticsearch package when switching to a different package type 21 | become: yes 22 | dpkg_selections: 23 | name: "{{ es_other_package_name }}" 24 | selection: "install" 25 | when: elasticsearch_package.stdout == 'install ok installed' 26 | 27 | - name: stop elasticsearch 28 | service: 29 | name: 'elasticsearch' 30 | state: stopped 31 | when: elasticsearch_package.stdout == 'install ok installed' 32 | 33 | - name: Debian - Remove elasticsearch package if we are switching to a different package type 34 | apt: 35 | name: '{{ es_other_package_name }}' 36 | state: absent 37 | when: elasticsearch_package.stdout == 'install ok installed' 38 | 39 | 40 | - name: Install Elasticsearch repository 41 | when: es_use_repository 42 | become: yes 43 | block: 44 | - name: Debian - Install apt-transport-https to support https APT downloads 45 | apt: 46 | name: apt-transport-https 47 | state: present 48 | 49 | - name: Debian - Add Elasticsearch repository key 50 | apt_key: 51 | url: '{{ es_apt_key }}' 52 | id: '{{ es_apt_key_id }}' 53 | state: present 54 | when: es_add_repository and es_apt_key | string 55 | 56 | - name: Debian - Add elasticsearch repository 57 | apt_repository: 58 | repo: '{{ item.repo }}' 59 | state: '{{ item.state }}' 60 | when: es_add_repository 61 | with_items: 62 | - { repo: "{{ es_apt_url_old }}", state: "absent" } 63 | - { repo: "{{ es_apt_url }}", state: "present" } 64 | - { repo: "{{ es_other_apt_url }}", state: "absent" } 65 | 66 | - name: Include optional user and group creation. 67 | when: (es_user_id is defined) and (es_group_id is defined) 68 | include: elasticsearch-optional-user.yml 69 | 70 | - name: Debian - Get installed elasticsearch version 71 | command: dpkg-query --showformat='${Version}' --show {{ es_package_name }} 72 | register: installed_es_version 73 | failed_when: False 74 | changed_when: False 75 | check_mode: no 76 | 77 | - name: Debian - unhold elasticsearch version 78 | become: yes 79 | dpkg_selections: 80 | name: "{{ es_package_name }}" 81 | selection: "install" 82 | when: not es_version_lock or (installed_es_version.stdout and installed_es_version.stdout != es_version) 83 | 84 | - name: Debian - Ensure elasticsearch is installed 85 | become: yes 86 | apt: 87 | name: '{{ es_package_name }}{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %}' 88 | state: present 89 | force: '{{ force_install }}' 90 | allow_unauthenticated: "{{ 'no' if es_apt_key else 'yes' }}" 91 | cache_valid_time: 86400 92 | when: es_use_repository 93 | register: debian_elasticsearch_install_from_repo 94 | notify: restart elasticsearch 95 | environment: 96 | ES_PATH_CONF: "{{ es_conf_dir }}" 97 | 98 | - name: Debian - hold elasticsearch version 99 | become: yes 100 | dpkg_selections: 101 | name: "{{ es_package_name }}" 102 | selection: "hold" 103 | when: es_version_lock 104 | 105 | # workaround due to https://github.com/ansible/ansible/issues/66977 106 | - set_fact: 107 | es_deb_url: "{% if es_version is version('7.0.0', '>=') %}{{ es_package_url }}-{{ es_version }}-amd64.deb{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %}" 108 | 109 | - name: Debian - Install Elasticsearch from url 110 | become: yes 111 | apt: 112 | deb: "{% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_deb_url }}{% endif %}" 113 | state: present 114 | when: not es_use_repository 115 | register: elasticsearch_install_from_package 116 | notify: restart elasticsearch 117 | environment: 118 | ES_PATH_CONF: "{{ es_conf_dir }}" 119 | -------------------------------------------------------------------------------- /tasks/elasticsearch-ssl.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: set fact es_same_keystore 4 | set_fact: es_same_keystore=false 5 | 6 | - name: set fact es_same_keystore if stores match 7 | set_fact: es_same_keystore=true 8 | when: es_ssl_keystore == es_ssl_truststore 9 | 10 | - name: Ensure certificate directory exists 11 | become: yes 12 | file: 13 | dest: "{{ es_ssl_certificate_path }}" 14 | state: directory 15 | owner: root 16 | group: "{{ es_group }}" 17 | mode: "750" 18 | when: es_ssl_upload 19 | 20 | - name: Upload SSL/TLS keystore 21 | become: yes 22 | copy: 23 | src: "{{ es_ssl_keystore }}" 24 | dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" 25 | owner: "{{ es_user }}" 26 | group: "{{ es_group }}" 27 | mode: "640" 28 | when: es_ssl_upload and es_ssl_keystore and es_ssl_truststore 29 | notify: restart elasticsearch 30 | register: copy_keystore 31 | 32 | - name: Upload SSL/TLS truststore 33 | become: yes 34 | copy: 35 | src: "{{ es_ssl_truststore }}" 36 | dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" 37 | owner: "{{ es_user }}" 38 | group: "{{ es_group }}" 39 | mode: "640" 40 | when: es_ssl_upload and es_ssl_keystore and es_ssl_truststore 41 | notify: restart elasticsearch 42 | register: copy_truststore 43 | 44 | - name: Upload SSL/TLS key and certificate 45 | become: yes 46 | copy: 47 | src: "{{ item }}" 48 | dest: "{{ es_ssl_certificate_path }}/{{ item | basename }}" 49 | owner: "{{ es_user }}" 50 | group: "{{ es_group }}" 51 | mode: "640" 52 | with_items: 53 | - "{{ es_ssl_key }}" 54 | - "{{ es_ssl_certificate }}" 55 | when: es_ssl_upload and es_ssl_key and es_ssl_certificate 56 | #Restart if these change 57 | notify: restart elasticsearch 58 | register: copy_certificates 59 | 60 | - name: Upload SSL Certificate Authority 61 | become: yes 62 | copy: 63 | src: "{{ es_ssl_certificate_authority }}" 64 | dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" 65 | owner: "{{ es_user }}" 66 | group: "{{ es_group }}" 67 | mode: "640" 68 | #Restart if this changes 69 | notify: restart elasticsearch 70 | when: es_ssl_upload and (es_ssl_certificate_authority is defined) and (es_ssl_certificate_authority|length > 0) 71 | 72 | - name: Set keystore password 73 | become: yes 74 | shell: echo "{{ es_ssl_keystore_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.keystore.secure_password' 75 | no_log: True 76 | when: es_ssl_keystore_password and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) 77 | with_items: 78 | - http 79 | - transport 80 | 81 | - name: Set truststore password 82 | become: yes 83 | shell: echo "{{ es_ssl_truststore_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.truststore.secure_password' 84 | no_log: True 85 | when: es_ssl_truststore_password and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) 86 | with_items: 87 | - http 88 | - transport 89 | 90 | - name: Remove keystore password 91 | become: yes 92 | shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.keystore.secure_password'" 93 | when: es_ssl_keystore_password == "" and (copy_keystore.changed or (es_same_keystore and copy_truststore.changed)) 94 | ignore_errors: yes 95 | with_items: 96 | - http 97 | - transport 98 | 99 | - name: Remove truststore password 100 | become: yes 101 | shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.truststore.secure_password'" 102 | when: es_ssl_truststore_password == "" and (copy_truststore.changed or (es_same_keystore and copy_keystore.changed)) 103 | ignore_errors: yes 104 | with_items: 105 | - http 106 | - transport 107 | 108 | - name: Set key password 109 | become: yes 110 | shell: echo "{{ es_ssl_key_password }}" | {{ es_home }}/bin/elasticsearch-keystore add -x -f 'xpack.security.{{ item }}.ssl.secure_key_passphrase' 111 | no_log: True 112 | when: es_ssl_key_password and copy_certificates.changed 113 | with_items: 114 | - http 115 | - transport 116 | 117 | - name: Remove key password 118 | become: yes 119 | shell: "{{ es_home }}/bin/elasticsearch-keystore remove 'xpack.security.{{ item }}.ssl.secure_key_passphrase'" 120 | when: es_ssl_key_password == "" and copy_certificates.changed 121 | ignore_errors: yes 122 | with_items: 123 | - http 124 | - transport 125 | -------------------------------------------------------------------------------- /docs/ssl-tls-setup.md: -------------------------------------------------------------------------------- 1 | # X-Pack Security SSL/TLS 2 | 3 | The role allows configuring HTTP and transport layer SSL/TLS for the cluster. You will need to generate and provide your own PKCS12 or PEM encoded certificates as described in [Encrypting communications in Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.4/configuring-tls.html#configuring-tls). 4 | 5 | By default this role will upload the certs to your elasticsearch servers. If you already copied the certs by your own way, set `es_ssl_upload` to `false` (default: `true`) 6 | 7 | If you don't want this role to add autogenerated SSL configuration to elasticsearch.yml set `es_enable_auto_ssl_configuration` to `false` (default: `true`). 8 | 9 | The following should be configured to ensure a security-enabled cluster successfully forms: 10 | 11 | * `es_enable_http_ssl` Default `false`. Setting this to `true` will enable HTTP client SSL/TLS 12 | * `es_enable_transport_ssl` - Default `false`. Setting this to `true` will enable transport layer SSL/TLS 13 | 14 | When using a [PKCS12](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#security-http-pkcs12-files) keystore and truststore: 15 | 16 | * `es_ssl_keystore` path to your PKCS12 keystore (can be the same as `es_ssl_truststore`) 17 | * `es_ssl_keystore_password` set this if your keystore is protected with a password 18 | * `es_ssl_truststore` path to your PKCS12 keystore (can be the same as `es_ssl_keystore`) 19 | * `es_ssl_truststore_password` set this if your truststore is protected with a password 20 | 21 | When using [PEM encoded](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#_pem_encoded_files_3) certificates: 22 | 23 | * `es_ssl_key` path to your SSL key 24 | * `es_ssl_key_password` set this if your SSL key is protected with a password 25 | * `es_ssl_certificate` the path to your SSL certificate 26 | 27 | ## Generating an SSL keystore 28 | 29 | With a password: 30 | 31 | ```shell 32 | $ bin/elasticsearch-certutil ca --out ./my-ca.p12 --pass "ca_password" 33 | $ bin/elasticsearch-certutil cert --ca ./my-ca.p12 --ca-pass "ca_password" --out ./my-keystore.p12 --pass "keystore_password" 34 | ``` 35 | 36 | Without a password: 37 | 38 | ```shell 39 | $ bin/elasticsearch-certutil ca --out ./my-ca.p12 --pass "" 40 | $ bin/elasticsearch-certutil cert --ca ./my-ca.p12 --out ./my-keystore.p12 --pass "" 41 | ``` 42 | 43 | ## Additional optional SSL/TLS configuration 44 | 45 | * `es_enable_auto_ssl_configuration` Default `true`. Whether this role should add automatically generated SSL config to elasticsearch.yml. 46 | * `es_ssl_certificate_path` Default `{{ es_conf_dir }}/certs`. The location where certificates should be stored on the ES node. 47 | * `es_ssl_verification_mode` Default `certificate`. See [SSL verification_mode](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#ssl-tls-settings) for options. 48 | * `es_ssl_certificate_authority` PEM encoded certificate file that should be trusted. 49 | * `es_validate_certs` Default `yes`. Determines if ansible should validate SSL certificates when performing actions over HTTPS. e.g. installing templates and managing native users. 50 | 51 | ## Example SSL/TLS configuration 52 | 53 | ```yaml 54 | - name: Elasticsearch with SSL/TLS enabled 55 | hosts: localhost 56 | roles: 57 | - role: elastic.elasticsearch 58 | vars: 59 | es_config: 60 | node.name: "node1" 61 | cluster.name: "custom-cluster" 62 | discovery.seed_hosts: "localhost:9301" 63 | http.port: 9201 64 | transport.port: 9301 65 | node.data: false 66 | node.master: true 67 | bootstrap.memory_lock: true 68 | xpack.security.authc.realms.file.file1.order: 0 69 | xpack.security.authc.realms.native.native1.order: 1 70 | es_heap_size: 1g 71 | es_api_basic_auth_username: "elastic" # This is the default user created by the installation of elasticsearch 72 | es_api_basic_auth_password: "changeme" # This is the default password created by the installation of elasticsearch 73 | es_enable_http_ssl: true 74 | es_enable_transport_ssl: true 75 | es_ssl_keystore: "files/certs/my-keystore.p12" 76 | es_ssl_truststore: "files/certs/my-ca.p12" 77 | es_ssl_keystore_password: "keystore_password" 78 | es_ssl_truststore_password: "ca_password" 79 | es_validate_certs: no 80 | ``` 81 | 82 | ## Changing the default password of elastic user 83 | 84 | To change the default password of user elastic: 85 | 86 | * Add this line to your playbook: 87 | 88 | ``` 89 | vars: 90 | es_api_basic_auth_username: "elastic" 91 | es_api_basic_auth_password: "changeme" 92 | es_users: 93 | native: 94 | elastic: 95 | password: "" 96 | ``` 97 | 98 | * Deploy your playbook 99 | * Update your playbook with: 100 | 101 | ``` 102 | vars: 103 | es_api_basic_auth_username: "elastic" 104 | es_api_basic_auth_password: "" 105 | ``` 106 | -------------------------------------------------------------------------------- /test/integration/files/custom_config/log4j2.properties: -------------------------------------------------------------------------------- 1 | {% raw %} 2 | # Log4j CUSTOM FILE 3 | 4 | status = error 5 | logger.action.name = org.elasticsearch.action 6 | logger.action.level = debug 7 | appender.rolling.type = Console 8 | appender.rolling.name = rolling 9 | appender.rolling.layout.type = ESJsonLayout 10 | appender.rolling.layout.type_name = server 11 | rootLogger.level = info 12 | rootLogger.appenderRef.rolling.ref = rolling 13 | appender.deprecation_rolling.type = Console 14 | appender.deprecation_rolling.name = deprecation_rolling 15 | appender.deprecation_rolling.layout.type = ESJsonLayout 16 | appender.deprecation_rolling.layout.type_name = deprecation 17 | appender.deprecation_rolling.layout.esmessagefields=x-opaque-id 18 | logger.deprecation.name = org.elasticsearch.deprecation 19 | logger.deprecation.level = warn 20 | logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling 21 | logger.deprecation.additivity = false 22 | appender.index_search_slowlog_rolling.type = Console 23 | appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling 24 | appender.index_search_slowlog_rolling.layout.type = ESJsonLayout 25 | appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog 26 | appender.index_search_slowlog_rolling.layout.esmessagefields=message,took,took_millis,total_hits,stats,search_type,total_shards,source,id 27 | logger.index_search_slowlog_rolling.name = index.search.slowlog 28 | logger.index_search_slowlog_rolling.level = trace 29 | logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling 30 | logger.index_search_slowlog_rolling.additivity = false 31 | appender.index_indexing_slowlog_rolling.type = Console 32 | appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling 33 | appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout 34 | appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog 35 | appender.index_indexing_slowlog_rolling.layout.esmessagefields=message,took,took_millis,doc_type,id,routing,source 36 | logger.index_indexing_slowlog.name = index.indexing.slowlog.index 37 | logger.index_indexing_slowlog.level = trace 38 | logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling 39 | logger.index_indexing_slowlog.additivity = false 40 | appender.audit_rolling.type = Console 41 | appender.audit_rolling.name = audit_rolling 42 | appender.audit_rolling.layout.type = PatternLayout 43 | appender.audit_rolling.layout.pattern = {\ 44 | "type": "audit", \ 45 | "timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}"\ 46 | %varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\ 47 | %varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\ 48 | %varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\ 49 | %varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\ 50 | %varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\ 51 | %varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\ 52 | %varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\ 53 | %varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\ 54 | %varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\ 55 | %varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\ 56 | %varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\ 57 | %varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\ 58 | %varsNotEmpty{, "user.roles":%map{user.roles}}\ 59 | %varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\ 60 | %varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\ 61 | %varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\ 62 | %varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\ 63 | %varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}\ 64 | %varsNotEmpty{, "request.method":"%enc{%map{request.method}}{JSON}"}\ 65 | %varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}\ 66 | %varsNotEmpty{, "request.id":"%enc{%map{request.id}}{JSON}"}\ 67 | %varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}\ 68 | %varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\ 69 | %varsNotEmpty{, "indices":%map{indices}}\ 70 | %varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\ 71 | %varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\ 72 | %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\ 73 | %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\ 74 | %varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\ 75 | }%n 76 | logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail 77 | logger.xpack_security_audit_logfile.level = info 78 | logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling 79 | logger.xpack_security_audit_logfile.additivity = false 80 | logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature 81 | logger.xmlsig.level = error 82 | logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter 83 | logger.samlxml_decrypt.level = fatal 84 | logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter 85 | logger.saml2_decrypt.level = fatal 86 | {% endraw %} 87 | -------------------------------------------------------------------------------- /docs/8x-support.md: -------------------------------------------------------------------------------- 1 | # 8.x support 2 | 3 | In [December 2021](https://github.com/elastic/ansible-elasticsearch/pull/838), we made the hard decision to deprecate this Ansible playbook without adding support for Elasticsearch 8.X. 4 | We acknowledge the impact this has had on many developers and organizations, and while we are not reverting the decision, we decided to share some guidelines around how to proceed from here, for folks wanting to keep using this playbook with Elasticsearch 8.X. 5 | 6 | --- 7 | 8 | At a high level, this role is expected to work in most cases for fresh installs and upgrades from 7.17+ by only overriding the `es_version` variable as long as the security is enforced properly using the [SSL/TLS doc](ssl-tls-setup.md). 9 | 10 | 1. install the last released version of the role from galaxy: `ansible-galaxy install elastic.elasticsearch,v7.17.0` 11 | 12 | 2. copy the TLS PKCS12 keystore and truststore (https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#security-http-pkcs12-files) 13 | 14 | 3. write a minimal playbook to deploy 8.2.3 on localhost: 15 | ```yaml 16 | - hosts: localhost 17 | roles: 18 | - elastic.elasticsearch 19 | vars: 20 | es_version: 8.2.3 21 | es_api_basic_auth_username: elastic 22 | es_api_basic_auth_password: changeme 23 | es_enable_http_ssl: true 24 | es_enable_transport_ssl: true 25 | es_ssl_keystore: "certs/keystore-password.p12" 26 | es_ssl_truststore: "certs/truststore-password.p12" 27 | es_ssl_keystore_password: password1 28 | es_ssl_truststore_password: password2 29 | es_validate_certs: no 30 | ``` 31 | 32 | 4. deploy locally: `ansible-playbook es.yml` 33 | 34 | ## Context for the below experiment 35 | 36 | The intent is to assess if the current playbook can still work with ES 8.X and what modifications may be needed. The testing was done on Ubuntu 20.04 and CentOS7 GCP VMs. 37 | 38 | The only code change done in the Ansible playbook was the override of the `es_version` variable. 39 | 40 | ### What is working 41 | 42 | - ✅ Deploying a standalone Elasticsearch cluster in 8.2.3 with the security example playbook from 7.x: 43 | - ✅ managing Elasticsearch users 44 | - ✅ upgrading a 7.17.0 standalone cluster **with security already enabled** to 8.2.3 45 | - ✅ managing Elasticsearch license 46 | 47 | The below configuration was used in the tests 48 | 49 | ```yaml 50 | - hosts: localhost 51 | roles: 52 | - elastic.elasticsearch 53 | vars: 54 | es_config: 55 | xpack.security.authc.realms.file.file1.order: 0 56 | es_api_basic_auth_username: elastic 57 | es_api_basic_auth_password: changeme 58 | es_api_sleep: 5 59 | es_enable_http_ssl: true 60 | es_enable_transport_ssl: true 61 | es_ssl_keystore: "test/integration/files/certs/keystore-password.p12" 62 | es_ssl_truststore: "test/integration/files/certs/truststore-password.p12" 63 | es_ssl_keystore_password: password1 64 | es_ssl_truststore_password: password2 65 | es_validate_certs: no 66 | es_users: 67 | file: 68 | es_admin: 69 | password: changeMe 70 | roles: 71 | - admin 72 | testUser: 73 | password: changeMeAlso! 74 | roles: 75 | - power_user 76 | - user 77 | es_roles: 78 | file: 79 | admin: 80 | cluster: 81 | - all 82 | indices: 83 | - names: '*' 84 | privileges: 85 | - all 86 | power_user: 87 | cluster: 88 | - monitor 89 | indices: 90 | - names: '*' 91 | privileges: 92 | - all 93 | user: 94 | indices: 95 | - names: '*' 96 | privileges: 97 | - read 98 | ``` 99 | 100 | ### What is not working 101 | 102 | **Deploying an 8.X cluster with the default Ansible configuration (no security) will not work.** 103 | 104 | When runnin Elasticsearch 8.x outside of Ansible without any security configuration, Elasticsearch will autogenerate a security configuration and still activate security. 105 | However, when you run Elasticsearch 8.x as part of the Ansible role without any security configuration, this will fail because the Ansible role will not be able to retrieve and use the autogenerated security configuration. 106 | 107 | To tackle this, you always have to specify your own security configuration based on the [SSL/TLS doc](ssl-tls-setup.md). 108 | 109 | 110 | ### What has not been tested 111 | 112 | **Deploying a 3 nodes cluster** 113 | 114 | When trying to deploy a 3 nodes clusters, the nodes seem to be configured successfully but they aren't able to communicate together with the test certificates (the ones used in automated standalone tests). It's highly likely that the problem lies with the tests certs themselves and not with the role. 115 | 116 | Should you be able to deploy a multi-node clusters, you will most likely have to change the configuration to use the new `node.roles` parameter ([example](https://github.com/elastic/ansible-elasticsearch/pull/772)) instead of the `node.master` and `node.data` (which got deprecated in 7.9, but the role never got [fixed](https://github.com/elastic/ansible-elasticsearch/issues/731). 117 | -------------------------------------------------------------------------------- /templates/jvm.options.j2: -------------------------------------------------------------------------------- 1 | ## JVM configuration 2 | 3 | ################################################################ 4 | ## IMPORTANT: JVM heap size 5 | ################################################################ 6 | ## 7 | ## You should always set the min and max JVM heap 8 | ## size to the same value. For example, to set 9 | ## the heap to 4 GB, set: 10 | ## 11 | ## -Xms4g 12 | ## -Xmx4g 13 | ## 14 | ## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html 15 | ## for more information 16 | ## 17 | ################################################################ 18 | 19 | # Xms represents the initial size of total heap space 20 | # Xmx represents the maximum size of total heap space 21 | {% if es_heap_size is defined %} 22 | -Xms{{ es_heap_size }} 23 | -Xmx{{ es_heap_size }} 24 | {% else %} 25 | -Xms2g 26 | -Xmx2g 27 | {% endif %} 28 | 29 | ################################################################ 30 | ## Expert settings 31 | ################################################################ 32 | ## 33 | ## All settings below this section are considered 34 | ## expert settings. Don't tamper with them unless 35 | ## you understand what you are doing 36 | ## 37 | ################################################################ 38 | 39 | ## GC configuration 40 | {% if es_version is version('7.6.0', '<') %} 41 | -XX:+UseConcMarkSweepGC 42 | -XX:CMSInitiatingOccupancyFraction=75 43 | -XX:+UseCMSInitiatingOccupancyOnly 44 | {% else %} 45 | 8-13:-XX:+UseConcMarkSweepGC 46 | 8-13:-XX:CMSInitiatingOccupancyFraction=75 47 | 8-13:-XX:+UseCMSInitiatingOccupancyOnly 48 | {% endif %} 49 | 50 | ## G1GC Configuration 51 | # NOTE: G1 GC is only supported on JDK version 10 or later 52 | {% if es_version is version('7.6.0', '<') %} 53 | # To use G1GC uncomment the lines below. 54 | # 10-:-XX:-UseConcMarkSweepGC 55 | # 10-:-XX:-UseCMSInitiatingOccupancyOnly 56 | # 10-:-XX:+UseG1GC 57 | {% if es_version is version('7.4.0', '<') %} 58 | # 10-:-XX:InitiatingHeapOccupancyPercent=75 59 | {% else %} 60 | # 10-:-XX:G1ReservePercent=25 61 | # 10-:-XX:InitiatingHeapOccupancyPercent=30 62 | {% endif %} 63 | {% else %} 64 | # to use G1GC, uncomment the next two lines and update the version on the 65 | # following three lines to your version of the JDK 66 | # 10-13:-XX:-UseConcMarkSweepGC 67 | # 10-13:-XX:-UseCMSInitiatingOccupancyOnly 68 | 14-:-XX:+UseG1GC 69 | 14-:-XX:G1ReservePercent=25 70 | 14-:-XX:InitiatingHeapOccupancyPercent=30 71 | {% endif %} 72 | 73 | {% if es_version is version('7.5.0', '<') %} 74 | ## DNS cache policy 75 | # cache ttl in seconds for positive DNS lookups noting that this overrides the 76 | # JDK security property networkaddress.cache.ttl; set to -1 to cache forever 77 | -Des.networkaddress.cache.ttl=60 78 | # cache ttl in seconds for negative DNS lookups noting that this overrides the 79 | # JDK security property networkaddress.cache.negative ttl; set to -1 to cache 80 | # forever 81 | -Des.networkaddress.cache.negative.ttl=10 82 | 83 | ## optimizations 84 | 85 | # pre-touch memory pages used by the JVM during initialization 86 | -XX:+AlwaysPreTouch 87 | 88 | ## basic 89 | 90 | # explicitly set the stack size 91 | -Xss1m 92 | 93 | # set to headless, just in case 94 | -Djava.awt.headless=true 95 | 96 | # ensure UTF-8 encoding by default (e.g. filenames) 97 | -Dfile.encoding=UTF-8 98 | 99 | # use our provided JNA always versus the system one 100 | -Djna.nosys=true 101 | 102 | # turn off a JDK optimization that throws away stack traces for common 103 | # exceptions because stack traces are important for debugging 104 | -XX:-OmitStackTraceInFastThrow 105 | 106 | # flags to configure Netty 107 | -Dio.netty.noUnsafe=true 108 | -Dio.netty.noKeySetOptimization=true 109 | -Dio.netty.recycler.maxCapacityPerThread=0 110 | {% if es_version is version('7.4.0', '>=') %} 111 | -Dio.netty.allocator.numDirectArenas=0 112 | {% endif %} 113 | 114 | # log4j 2 115 | -Dlog4j.shutdownHookEnabled=false 116 | -Dlog4j2.disable.jmx=true 117 | {% endif %} 118 | 119 | ## JVM temporary directory 120 | -Djava.io.tmpdir=${ES_TMPDIR} 121 | 122 | ## heap dumps 123 | 124 | # generate a heap dump when an allocation from the Java heap fails 125 | # heap dumps are created in the working directory of the JVM 126 | -XX:+HeapDumpOnOutOfMemoryError 127 | 128 | # specify an alternative path for heap dumps; ensure the directory exists and 129 | # has sufficient space 130 | -XX:HeapDumpPath={{ es_heap_dump_path }} 131 | 132 | # specify an alternative path for JVM fatal error logs 133 | -XX:ErrorFile={{ es_log_dir }}/hs_err_pid%p.log 134 | 135 | ## JDK 8 GC logging 136 | 137 | 8:-XX:+PrintGCDetails 138 | 8:-XX:+PrintGCDateStamps 139 | 8:-XX:+PrintTenuringDistribution 140 | 8:-XX:+PrintGCApplicationStoppedTime 141 | 8:-Xloggc:{{ es_log_dir }}/gc.log 142 | 8:-XX:+UseGCLogFileRotation 143 | 8:-XX:NumberOfGCLogFiles=32 144 | 8:-XX:GCLogFileSize=64m 145 | 146 | # JDK 9+ GC logging 147 | 9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ es_log_dir }}/gc.log:utctime,pid,tags:filecount=32,filesize=64m 148 | {% if es_version is version('7.5.0', '<') %} 149 | # due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise 150 | # time/date parsing will break in an incompatible way for some date patterns and locals 151 | 9-:-Djava.locale.providers=COMPAT 152 | 153 | {% if es_major_version == "6.x" %} 154 | # temporary workaround for C2 bug with JDK 10 on hardware with AVX-512 155 | 10-:-XX:UseAVX=2 156 | {% endif %} 157 | {% endif %} 158 | 159 | {% if es_jvm_custom_parameters !='' %} 160 | {% for item in es_jvm_custom_parameters %} 161 | {{ item }} 162 | {% endfor %} 163 | {% endif %} 164 | -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/shared_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'json' 3 | vars = JSON.parse(File.read('/tmp/vars.json')) 4 | 5 | $families = { 6 | 'Debian' => { 7 | 'shell' => '/bin/false', 8 | 'password' => '*', 9 | 'defaults_path' => '/etc/default/elasticsearch' 10 | }, 11 | 'RedHat' => { 12 | 'shell' => '/sbin/nologin', 13 | 'password' => '!!', 14 | 'defaults_path' => '/etc/sysconfig/elasticsearch' 15 | } 16 | } 17 | 18 | $family = $families[vars['ansible_os_family']] 19 | 20 | es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" 21 | username = vars['es_api_basic_auth_username'] 22 | password = vars['es_api_basic_auth_password'] 23 | 24 | # Sample of default features status 25 | features = { 26 | 'monitoring' => { 27 | 'enabled' => 'true', 28 | 'available' => 'true' 29 | }, 30 | 'ml' => { 31 | 'enabled' => 'true', 32 | 'available' => 'false' 33 | }, 34 | 'sql' => { 35 | 'enabled' => 'true', 36 | 'available' => 'true' 37 | } 38 | } 39 | 40 | shared_examples 'shared::init' do |vars| 41 | describe 'version check' do 42 | it 'should be reported as version '+vars['es_version'] do 43 | expect(curl_json(es_api_url, username=username, password=password)['version']['number']).to eq(vars['es_version']) 44 | end 45 | end 46 | describe 'xpack checks' do 47 | if not vars['oss_version'] 48 | it 'should be be running the basic version' do 49 | expect(curl_json("#{es_api_url}/_xpack", username=username, password=password)['tagline']).to eq('You know, for X') 50 | end 51 | it 'xpack should be activated' do 52 | expect(curl_json("#{es_api_url}/_license", username=username, password=password)['license']['status']).to eq('active') 53 | end 54 | end 55 | end 56 | describe user(vars['es_user']) do 57 | it { should exist } 58 | it { should belong_to_group vars['es_group'] } 59 | it { should have_uid vars['es_user_id'] } if vars.key?('es_user_id') 60 | 61 | it { should have_login_shell $family['shell'] } 62 | 63 | its(:encrypted_password) { should eq($family['password']) } 64 | end 65 | 66 | describe package(vars['es_package_name']) do 67 | it { should be_installed } 68 | end 69 | 70 | describe service("elasticsearch") do 71 | it { should be_running } 72 | end 73 | 74 | describe port(vars['es_api_port']) do 75 | it { should be_listening.with('tcp') } 76 | end 77 | 78 | if vars['es_templates'] 79 | describe file('/etc/elasticsearch/templates') do 80 | it { should be_directory } 81 | it { should be_owned_by 'root' } 82 | end 83 | describe file('/etc/elasticsearch/templates/basic.json') do 84 | it { should be_file } 85 | it { should be_owned_by 'root' } 86 | end 87 | #This is possibly subject to format changes in the response across versions so may fail in the future 88 | describe 'Template Contents Correct' do 89 | it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do 90 | template = curl_json("#{es_api_url}/_template/basic", username=username, password=password) 91 | expect(template.key?('basic')) 92 | expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") 93 | if vars['es_major_version'] == '7.x' 94 | expect(template['basic']['mappings']['_source']['enabled']).to eq(false) 95 | else 96 | expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) 97 | end 98 | end 99 | end 100 | end 101 | 102 | describe file($family['defaults_path']) do 103 | its(:content) { should match '' } 104 | end 105 | 106 | if vars.key?('es_plugins') 107 | vars['es_plugins'].each do |plugin| 108 | name = plugin['plugin'] 109 | describe file('/usr/share/elasticsearch/plugins/'+name) do 110 | it { should be_directory } 111 | it { should be_owned_by 'root' } 112 | end 113 | it 'should be installed and the right version' do 114 | plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password) 115 | version = nil 116 | _node, data = plugins['nodes'].first 117 | data['plugins'].each do |p| 118 | version = p['version'] if p['name'] == name 119 | end 120 | expect(version).to eql(vars['es_version']) 121 | end 122 | end 123 | end 124 | describe file("/etc/elasticsearch/elasticsearch.yml") do 125 | it { should be_owned_by 'root' } 126 | it { should contain "node.name: localhost" } 127 | it { should contain 'cluster.name: elasticsearch' } 128 | it { should_not contain "path.conf: /etc/elasticsearch" } 129 | its(:content) { should match "path.data: #{vars['es_data_dirs'].join(',')}" } 130 | its(:content) { should match "path.logs: /var/log/elasticsearch" } 131 | end 132 | 133 | if vars['es_use_repository'] 134 | if vars['ansible_os_family'] == 'RedHat' 135 | describe file("/etc/yum.repos.d/elasticsearch-#{vars['es_repo_name']}.repo") do 136 | it { should exist } 137 | end 138 | describe yumrepo("elasticsearch-#{vars['es_repo_name']}") do 139 | it { should exist } 140 | it { should be_enabled } 141 | end 142 | describe file("/etc/yum.repos.d/elasticsearch-#{vars['es_other_repo_name']}.repo") do 143 | it { should_not exist } 144 | end 145 | describe yumrepo("elasticsearch-#{vars['es_other_repo_name']}") do 146 | it { should_not exist } 147 | it { should_not be_enabled } 148 | end 149 | end 150 | if vars['ansible_os_family'] == 'Debian' 151 | describe command('apt-cache policy') do 152 | its(:stdout) { should match /elastic.co.*\/#{Regexp.quote(vars['es_repo_name'])}\//} 153 | its(:stdout) { should_not match /elastic.co.*\/#{Regexp.quote(vars['es_other_repo_name'])}\//} 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /docs/multi-instance.md: -------------------------------------------------------------------------------- 1 | # Multi-instance Support 2 | 3 | Starting with ansible-elasticsearch:7.1.1, installing more than one instance of Elasticsearch **on the same host** is no longer supported. 4 | 5 | See [554#issuecomment-496804929](https://github.com/elastic/ansible-elasticsearch/issues/554#issuecomment-496804929) for more details about why we removed it. 6 | 7 | ## Upgrade procedure 8 | 9 | If you have single-instances hosts and want to upgrade from previous versions of the role: 10 | 11 | ### Procedure with data move 12 | 13 | This procedure will allow you to move your data to the new standard paths (see [#581](https://github.com/elastic/ansible-elasticsearch/issues/581)): 14 | 15 | 1. Stop Elasticsearch before the migration 16 | 17 | 2. Migrate your data to the new standard paths: 18 | ``` 19 | # mv /etc/elasticsearch/${ES_INSTANCE_NAME}/* /etc/elasticsearch/ && rm -fr /etc/elasticsearch/${ES_INSTANCE_NAME}/ 20 | mv: overwrite '/etc/elasticsearch/elasticsearch.keystore'? y 21 | # mv /var/lib/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/* /var/lib/elasticsearch/ && rm -fr /var/lib/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/ 22 | # ls /var/lib/elasticsearch/ 23 | nodes 24 | # mv /var/log/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/* /var/log/elasticsearch/ && rm -fr /var/log/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/ 25 | # rm -fr /var/run/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/ 26 | ``` 27 | 28 | 3. Update playbook (remove `es_conf_dir`, `es_data_dirs`, `es_log_dir`, `es_pid_dir` and `es_instance_name` variables) 29 | 30 | 4. Update ansible-role to new version ([7.1.1](https://github.com/elastic/ansible-elasticsearch/releases/tag/7.1.1) at the time of writing) and deploy ansible-role 31 | 32 | 5. After ansible-role new deployment, you can do some cleanup of old Init file and Default file: 33 | 34 | Example: 35 | ``` 36 | $ systemctl stop elasticsearch 37 | $ mv /etc/elasticsearch/${ES_INSTANCE_NAME}/* /etc/elasticsearch/ && rm -fr /etc/elasticsearch/${ES_INSTANCE_NAME}/ 38 | mv: overwrite '/etc/elasticsearch/elasticsearch.keystore'? y 39 | $ mv /var/lib/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/* /var/lib/elasticsearch/ && rm -fr /var/lib/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/ 40 | $ ls /var/lib/elasticsearch/ 41 | nodes 42 | $ mv /var/log/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/* /var/log/elasticsearch/ && rm -fr /var/log/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/ 43 | $ rm -fr /var/run/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}/ 44 | $ ansible-galaxy install --force elastic.elasticsearch,7.1.1 45 | - changing role elastic.elasticsearch from 6.6.0 to 7.1.1 46 | - downloading role 'elasticsearch', owned by elastic 47 | - downloading role from https://github.com/elastic/ansible-elasticsearch/archive/7.1.1.tar.gz 48 | - extracting elastic.elasticsearch to /home/jmlrt/.ansible/roles/elastic.elasticsearch 49 | - elastic.elasticsearch (7.1.1) was installed successfully 50 | $ ansible-playbook playbook.yml 51 | 52 | ... 53 | 54 | TASK [elastic.elasticsearch : Create Directories] 55 | ok: [localhost] => (item=/var/run/elasticsearch) 56 | ok: [localhost] => (item=/var/log/elasticsearch) 57 | changed: [localhost] => (item=/etc/elasticsearch) 58 | ok: [localhost] => (item=/var/lib/elasticsearch) 59 | 60 | TASK [elastic.elasticsearch : Copy Configuration File] 61 | changed: [localhost] 62 | 63 | TASK [elastic.elasticsearch : Copy Default File] 64 | changed: [localhost] 65 | 66 | TASK [elastic.elasticsearch : Copy jvm.options File] 67 | changed: [localhost] 68 | 69 | ... 70 | 71 | RUNNING HANDLER [elastic.elasticsearch : restart elasticsearch] 72 | changed: [localhost] 73 | 74 | ... 75 | 76 | PLAY RECAP 77 | localhost : ok=26 changed=6 unreachable=0 failed=0 skipped=116 rescued=0 ignored=0 78 | $ find /etc -name '${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME}*' 79 | /etc/default/node1_elasticsearch 80 | /etc/systemd/system/multi-user.target.wants/node1_elasticsearch.service 81 | ``` 82 | 83 | ### Procedure without data move 84 | 85 | This procedure will allow you to keep your data to the old paths: 86 | 87 | 1. Override these variables to match previous values: 88 | ```yaml 89 | es_conf_dir: /etc/elasticsearch/${ES_INSTANCE_NAME} 90 | es_data_dirs: 91 | - /var/lib/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME} 92 | es_log_dir: /var/log/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME} 93 | es_pid_dir: /var/run/elasticsearch/${INVENTORY_HOSTNAME}-${ES_INSTANCE_NAME} 94 | ``` 95 | 96 | 2. Deploy ansible-role. **Even if these variables are overrided, Elasticsearch config file and default option file will change, which imply an Elasticsearch restart.** 97 | 98 | 3. After ansible-role new deployment, you can do some cleanup of old Init file and Default file. 99 | 100 | Example: 101 | ```bash 102 | $ ansible-playbook -e '{"es_conf_dir":"/etc/elasticsearch/node1","es_data_dirs":["/var/lib/elasticsearch/localhost-node1"],"es_log_dir":"/var/log/elasticsearch/localhost-node1","es_pid_dir":"/var/run/elasticsearch/localhost-node1"}' playbook.yml 103 | ... 104 | TASK [elasticsearch : Create Directories] ********************************************************************************************************************************************************************************************************************** 105 | ok: [localhost] => (item=/var/run/elasticsearch/localhost-node1) 106 | ok: [localhost] => (item=/var/log/elasticsearch/localhost-node1) 107 | ok: [localhost] => (item=/etc/elasticsearch/node1) 108 | ok: [localhost] => (item=/var/lib/elasticsearch/localhost-node1) 109 | 110 | TASK [elasticsearch : Copy Configuration File] ***************************************************************************************************************************************************************************************************************** 111 | changed: [localhost] 112 | 113 | TASK [elasticsearch : Copy Default File] *********************************************************************************************************************************************************************************************************************** 114 | changed: [localhost] 115 | ... 116 | PLAY RECAP ***************************************************************************************************************************************************************************************************************************************************** 117 | localhost : ok=32 changed=3 unreachable=0 failed=0 118 | 119 | $ find /etc -name 'node1_elasticsearch*' 120 | /etc/default/node1_elasticsearch 121 | /etc/systemd/system/multi-user.target.wants/node1_elasticsearch.service 122 | $ rm /etc/default/node1_elasticsearch /etc/systemd/system/multi-user.target.wants/node1_elasticsearch.service 123 | ``` 124 | 125 | ## Workaround 126 | 127 | If you use more than one instance of Elasticsearch on the same host (with different ports, directory and config files), you are still be able to install Elasticsearch 6.x and 7.x in multi-instance mode by using ansible-elasticsearch commit [25bd09f](https://github.com/elastic/ansible-elasticsearch/commit/25bd09f6835b476b6a078676a7d614489a6739c5) (last commit before multi-instance removal) and overriding `es_version` variable: 128 | 129 | ```sh 130 | $ cat << EOF >> requirements.yml # require git 131 | - src: https://github.com/elastic/ansible-elasticsearch 132 | version: 25bd09f 133 | name: elasticsearch 134 | EOF 135 | $ ansible-galaxy install -r requirements.yml 136 | $ cat << EOF >> playbook.yml 137 | - hosts: localhost 138 | roles: 139 | - role: elasticsearch 140 | vars: 141 | es_instance_name: "node1" 142 | es_version: 7.1.1 # or 6.8.0 for example 143 | EOF 144 | $ ansible-playbook playbook.yml 145 | ``` 146 | -------------------------------------------------------------------------------- /tasks/xpack/security/elasticsearch-security-native.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set fact change_api_password to false 3 | set_fact: change_api_password=false 4 | 5 | - name: set fact manage_native_users to false 6 | set_fact: manage_native_users=false 7 | 8 | - name: set fact manage_native_users to true 9 | set_fact: manage_native_users=true 10 | when: es_users is defined and es_users.native is defined and es_users.native.keys() | list | length > 0 11 | 12 | - name: set fact manage_native_role to false 13 | set_fact: manage_native_roles=false 14 | 15 | - name: set fact manage_native_roles to true 16 | set_fact: manage_native_roles=true 17 | when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | list | length > 0 18 | 19 | #If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load 20 | 21 | #List current users 22 | - name: List Native Users 23 | uri: 24 | url: "{{ es_api_uri }}/{{ es_security_api }}/user" 25 | method: GET 26 | user: "{{es_api_basic_auth_username}}" 27 | password: "{{es_api_basic_auth_password}}" 28 | force_basic_auth: yes 29 | status_code: 200 30 | validate_certs: "{{ es_validate_certs }}" 31 | register: user_list_response 32 | when: manage_native_users 33 | check_mode: no 34 | 35 | - name: set fact reserved_users equals user_list_response.json 36 | set_fact: reserved_users={{ user_list_response.json | filter_reserved }} 37 | when: manage_native_users 38 | 39 | #Current users not inc. those reserved 40 | - name: set fact current_users equals user_list_response.json.keys not including reserved 41 | set_fact: current_users={{ user_list_response.json.keys() | list | difference (reserved_users) }} 42 | when: manage_native_users 43 | 44 | #We are changing the es_api_basic_auth_username password, so we need to do it first and update the param 45 | - name: set fact native_users 46 | set_fact: native_users={{ es_users.native }} 47 | when: manage_native_users 48 | 49 | - name: set fact change_api_password to true 50 | set_fact: change_api_password=true 51 | when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined 52 | 53 | - name: Update API User Password 54 | uri: 55 | url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{es_api_basic_auth_username}}/_password" 56 | method: POST 57 | body_format: json 58 | body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }" 59 | status_code: 200 60 | user: "{{es_api_basic_auth_username}}" 61 | password: "{{es_api_basic_auth_password}}" 62 | force_basic_auth: yes 63 | validate_certs: "{{ es_validate_certs }}" 64 | when: change_api_password 65 | 66 | - name: set fact es_api_basic_auth_password 67 | set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}} 68 | when: change_api_password 69 | 70 | #Identify users that are present in ES but not declared and thus should be removed 71 | - name: set fact users_to_remove 72 | set_fact: users_to_remove={{ current_users | difference ( native_users.keys() | list) }} 73 | when: manage_native_users 74 | 75 | #Delete all non required users NOT inc. reserved 76 | - name: Delete Native Users 77 | uri: 78 | url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{item}}" 79 | method: DELETE 80 | status_code: 200 81 | user: "{{es_api_basic_auth_username}}" 82 | password: "{{es_api_basic_auth_password}}" 83 | force_basic_auth: yes 84 | validate_certs: "{{ es_validate_certs }}" 85 | when: manage_native_users and es_delete_unmanaged_native 86 | with_items: "{{ users_to_remove | default([]) }}" 87 | 88 | - name: set fact users_to_ignore 89 | set_fact: users_to_ignore={{ native_users.keys() | list | intersect (reserved_users) }} 90 | when: manage_native_users 91 | 92 | - name: debug message 93 | debug: 94 | msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}" 95 | when: manage_native_users and users_to_ignore | length > 0 96 | 97 | #Update password on all reserved users 98 | - name: Update Reserved User Passwords 99 | uri: 100 | url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{ item | urlencode }}/_password" 101 | method: POST 102 | body_format: json 103 | body: "{ \"password\":\"{{native_users[item].password}}\" }" 104 | status_code: 200 105 | user: "{{es_api_basic_auth_username}}" 106 | password: "{{es_api_basic_auth_password}}" 107 | force_basic_auth: yes 108 | validate_certs: "{{ es_validate_certs }}" 109 | when: native_users[item].password is defined 110 | no_log: True 111 | with_items: "{{ users_to_ignore | default([]) }}" 112 | 113 | - name: set fact users_to_modify 114 | set_fact: users_to_modify={{ native_users.keys() | list | difference (reserved_users) }} 115 | when: manage_native_users 116 | 117 | #Overwrite all other users NOT inc. those reserved 118 | - name: Update Non-Reserved Native User Details 119 | uri: 120 | url: "{{ es_api_uri }}/{{ es_security_api }}/user/{{ item | urlencode }}" 121 | method: POST 122 | body_format: json 123 | body: "{{ native_users[item] | to_json }}" 124 | status_code: 200 125 | user: "{{es_api_basic_auth_username}}" 126 | password: "{{es_api_basic_auth_password}}" 127 | force_basic_auth: yes 128 | validate_certs: "{{ es_validate_certs }}" 129 | when: manage_native_users 130 | no_log: True 131 | with_items: "{{ users_to_modify | default([]) }}" 132 | 133 | ## ROLE CHANGES 134 | 135 | #List current roles not. inc those reserved 136 | - name: List Native Roles 137 | uri: 138 | url: "{{ es_api_uri }}/{{ es_security_api }}/role" 139 | method: GET 140 | user: "{{es_api_basic_auth_username}}" 141 | password: "{{es_api_basic_auth_password}}" 142 | force_basic_auth: yes 143 | status_code: 200 144 | validate_certs: "{{ es_validate_certs }}" 145 | register: role_list_response 146 | when: manage_native_roles 147 | check_mode: no 148 | 149 | - name: set fact reserved roles 150 | set_fact: reserved_roles={{ role_list_response.json | filter_reserved }} 151 | when: manage_native_roles 152 | 153 | - name: set fact current roles 154 | set_fact: current_roles={{ role_list_response.json.keys() | list | difference (reserved_roles) }} 155 | when: manage_native_roles 156 | 157 | - name: set fact roles to ignore 158 | set_fact: roles_to_ignore={{ es_roles.native.keys() | list | intersect (reserved_roles) | default([]) }} 159 | when: manage_native_roles 160 | 161 | - name: debug message 162 | debug: 163 | msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}" 164 | when: manage_native_roles and roles_to_ignore | length > 0 165 | 166 | - name: set fact roles_to_remove 167 | set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() | list) }} 168 | when: manage_native_roles 169 | 170 | #Delete all non required roles NOT inc. reserved 171 | - name: Delete Native Roles 172 | uri: 173 | url: "{{ es_api_uri }}/{{ es_security_api }}/role/{{ item | urlencode }}" 174 | method: DELETE 175 | status_code: 200 176 | user: "{{es_api_basic_auth_username}}" 177 | password: "{{es_api_basic_auth_password}}" 178 | force_basic_auth: yes 179 | validate_certs: "{{ es_validate_certs }}" 180 | when: manage_native_roles and es_delete_unmanaged_native 181 | with_items: "{{roles_to_remove | default([]) }}" 182 | 183 | - name: set fact roles_to_modify 184 | set_fact: roles_to_modify={{ es_roles.native.keys() | list | difference (reserved_roles) }} 185 | when: manage_native_roles 186 | 187 | #Update other roles - NOT inc. reserved roles 188 | - name: Update Native Roles 189 | uri: 190 | url: "{{ es_api_uri }}/{{ es_security_api }}/role/{{ item | urlencode }}" 191 | method: POST 192 | body_format: json 193 | body: "{{ es_roles.native[item] | to_json}}" 194 | status_code: 200 195 | user: "{{es_api_basic_auth_username}}" 196 | password: "{{es_api_basic_auth_password}}" 197 | force_basic_auth: yes 198 | validate_certs: "{{ es_validate_certs }}" 199 | when: manage_native_roles 200 | with_items: "{{ roles_to_modify | default([]) }}" 201 | -------------------------------------------------------------------------------- /.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: docker 4 | 5 | transport: 6 | max_ssh_sessions: 6 7 | 8 | provisioner: 9 | name: ansible_playbook 10 | hosts: localhost 11 | roles_path: ../ 12 | require_ansible_repo: true 13 | require_ansible_omnibus: false 14 | require_ansible_source: false 15 | http_proxy: <%= ENV['HTTP_PROXY'] %> 16 | https_proxy: <%= ENV['HTTPS_PROXY'] %> 17 | no_proxy: localhost,127.0.0.1 18 | ignore_extensions_from_root: [".git",".idea",".kitchen.yml"] 19 | ignore_paths_from_root: [".git",".idea",".kitchen"] 20 | <% if ENV['VERSION'] %> 21 | attributes: 22 | extra_vars: 23 | es_major_version: "<%= ENV['VERSION'] %>" 24 | <% if ENV['VERSION'] == '6.x' %> 25 | es_version: '6.8.23' 26 | <% end %> 27 | <% end %> 28 | 29 | platforms: 30 | - name: ubuntu-14.04 31 | driver_config: 32 | image: ubuntu:14.04 33 | privileged: true 34 | provision_command: 35 | - apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa 36 | - apt-get update -q && apt-get -y -q install ansible openjdk-8-jre python-jmespath 37 | - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 38 | use_sudo: false 39 | volume: 40 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 41 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 42 | - name: ubuntu-16.04 43 | driver_config: 44 | image: ubuntu:16.04 45 | privileged: true 46 | provision_command: 47 | - apt-get update -q && apt-get install -y -q iproute locales software-properties-common && add-apt-repository -y ppa:ansible/ansible 48 | - apt-get update -q && apt-get install -y -q ansible python-jmespath 49 | - locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8 50 | use_sudo: false 51 | volume: 52 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 53 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 54 | run_command: "/sbin/init" 55 | - name: ubuntu-18.04 56 | driver_config: 57 | image: ubuntu:18.04 58 | privileged: true 59 | provision_command: 60 | - apt-get update -q && apt-get install -y -q ansible iproute2 python-jmespath 61 | use_sudo: false 62 | volume: 63 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 64 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 65 | run_command: "/sbin/init" 66 | - name: ubuntu-20.04 67 | driver_config: 68 | image: ubuntu:20.04 69 | privileged: true 70 | provision_command: 71 | - apt-get update -q && apt-get install -y -q ansible gpg iproute2 python3-jmespath 72 | use_sudo: false 73 | volume: 74 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 75 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 76 | run_command: "/sbin/init" 77 | - name: debian-8 78 | driver_config: 79 | image: debian:8 80 | privileged: true 81 | provision_command: 82 | - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath 83 | - echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list 84 | - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf 85 | - apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless 86 | - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list 87 | - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 88 | - apt-get update -q && apt-get install -y -q ansible 89 | volume: 90 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 91 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 92 | use_sudo: false 93 | run_command: "/sbin/init" 94 | - name: debian-9 95 | driver_config: 96 | image: debian:9 97 | privileged: true 98 | provision_command: 99 | - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath systemd-sysv 100 | - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list 101 | - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 102 | - apt-get update -q && apt-get install -y -q ansible 103 | volume: 104 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 105 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 106 | use_sudo: false 107 | run_command: "/sbin/init" 108 | - name: debian-10 109 | driver_config: 110 | image: debian:10 111 | privileged: true 112 | provision_command: 113 | - apt-get update -q && apt-get install -y -q gnupg2 python-jmespath systemd-sysv 114 | - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list 115 | - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 116 | - apt-get update -q && apt-get install -y -q ansible 117 | volume: 118 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 119 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 120 | use_sudo: false 121 | run_command: "/sbin/init" 122 | - name: centos-7 123 | driver_config: 124 | image: centos:7 125 | provision_command: 126 | - yum -y install epel-release 127 | - yum -y install ansible iproute python2-jmespath 128 | volume: 129 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 130 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 131 | run_command: "/usr/sbin/init" 132 | privileged: true 133 | use_sudo: false 134 | - name: centos-8 135 | driver_config: 136 | image: centos:8 137 | provision_command: 138 | - yum -y install epel-release 139 | - yum -y install ansible iproute python3-jmespath 140 | volume: 141 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 142 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 143 | run_command: "/usr/sbin/init" 144 | privileged: true 145 | use_sudo: false 146 | - name: amazonlinux-2 147 | driver_config: 148 | image: amazonlinux:2 149 | provision_command: 150 | - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 151 | - yum -y install ansible iproute python2-jmespath 152 | volume: 153 | - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json 154 | - /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers 155 | run_command: "/usr/sbin/init" 156 | privileged: true 157 | use_sudo: false 158 | 159 | suites: 160 | - name: custom-config 161 | provisioner: 162 | idempotency_test: true 163 | playbook: test/integration/custom-config.yml 164 | - name: default 165 | provisioner: 166 | playbook: test/integration/default.yml 167 | idempotency_test: true 168 | - name: license 169 | provisioner: 170 | playbook: test/integration/license.yml 171 | idempotency_test: true 172 | - name: oss 173 | provisioner: 174 | idempotency_test: true 175 | playbook: test/integration/oss.yml 176 | - name: oss-to-default-upgrade 177 | provisioner: 178 | playbook: test/integration/oss-to-default-upgrade.yml 179 | idempotency_test: false 180 | - name: oss-upgrade 181 | provisioner: 182 | playbook: test/integration/oss-upgrade.yml 183 | idempotency_test: false 184 | - name: security 185 | provisioner: 186 | playbook: test/integration/security.yml 187 | idempotency_test: true 188 | - name: trial 189 | provisioner: 190 | playbook: test/integration/trial.yml 191 | idempotency_test: false # es_xpack_trial is not idempotent currently 192 | - name: upgrade 193 | provisioner: 194 | playbook: test/integration/upgrade.yml 195 | idempotency_test: false 196 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ARCHIVED 2 | 3 | This project is no longer maintained. 4 | 5 | You are welcomed to keep using it and adapting it to work for your own needs, including with Elasticsearch [8.x](docs/8x-support.md). 6 | 7 | For alternative getting started experiences, you may want to try one of these options: 8 | 9 | - Start a [free trial on Elastic Cloud](https://www.elastic.co/cloud/elasticsearch-service/signup), our hosted service. 10 | - Take a look at [Elastic Cloud on Kubernetes (ECK)](https://elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html) for launching the stack via Kubernetes. 11 | - Read our [Running the Elastic Stack on Docker](https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-docker.html) guide. 12 | - Take a look at the [Elastic Stack Terraform provider.](https://github.com/elastic/terraform-provider-elasticstack) 13 | 14 | # ansible-elasticsearch 15 | [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/) 16 | 17 | 18 | **THIS ROLE IS FOR 7.x & 6.x**, but should still work with 8.x (see [note](docs/8x-support.md)). 19 | 20 | Ansible role for 7.x/6.x Elasticsearch - tests used to run and pass on the below platforms: 21 | 22 | * Ubuntu 16.04 23 | * Ubuntu 18.04 24 | * Ubuntu 20.04 25 | * Debian 8 26 | * Debian 9 27 | * Debian 10 28 | * CentOS 7 29 | * Amazon Linux 2 30 | 31 | ## BREAKING CHANGES 32 | 33 | ### Notice about multi-instance support 34 | 35 | * If you use only one instance but want to upgrade from an older ansible-elasticsearch version, follow [upgrade procedure](https://github.com/elastic/ansible-elasticsearch/blob/main/docs/multi-instance.md#upgrade-procedure) 36 | * If you install more than one instance of Elasticsearch on the same host (with different ports, directory and config files), **do not update to ansible-elasticsearch >= 7.1.1**, please follow this [workaround](https://github.com/elastic/ansible-elasticsearch/blob/main/docs/multi-instance.md#workaround) instead. 37 | * For multi-instances use cases, we are now recommending Docker containers using our official images (https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html). 38 | 39 | ### Removing the MAX_THREAD settings 40 | 41 | Ansible-elasticsearch 7.5.2 is removing the option to customize the maximum number of threads the process can start in [#637](https://github.com/elastic/ansible-elasticsearch/pull/637/files#diff-04c6e90faac2675aa89e2176d2eec7d8L408). 42 | We discovered that this option wasn't working anymore since multi-instance support removal in ansible-elasticsearch 7.1.1. 43 | This option will be added back in a following release if it's still relevant regarding latest Elasticsearch evolutions. 44 | 45 | ### Changes about configuration files 46 | 47 | Ansible-elasticsearch 7.5.2 is updating the configuration files provided by this role in [#637](https://github.com/elastic/ansible-elasticsearch/pull/637) which contained some options deprecated in 6.x and 7.x: 48 | - `/etc/default/elasticsearch`|`/etc/sysconfig/elasticsearch`: the new template reflect the configuration file provided by Elasticsearch >= 6.x, the parameters we removed were already not used in 6.x and 7.x 49 | - `/etc/elasticsearch/jvm.options`: the new template reflect the configuration files provided by Elasticsearch >= 6.x 50 | - `/etc/elasticsearch/log4j2.properties`: 51 | - We removed `log4j2.properties.j2` template from this Ansible role as it was a static file not bringing any customization specific to some ansible variable. 52 | - Deployment of this Ansible role on new servers will get the default `log4j2.properties` provided by Elasticsearch without any override. 53 | - **WARNING**: For upgrade scenarios where this file was already managed by previous versions of ansible-elasticsearch, this file will become unmanaged and won't be updated by default. If you wish to update it to 7.5 version, you can retrieve it [here](https://github.com/elastic/elasticsearch/blob/7.5/distribution/src/config/log4j2.properties) and use this file with `es_config_log4j2` Ansible variable (see below). 54 | 55 | ### Removing OSS distribution for versions >= 7.11.0 56 | 57 | Starting from Elasticsearch 7.11.0, OSS distributions will no longer be provided following the recent Elasticsearch license change. 58 | 59 | This Ansible role will fail if `oss_version` is set to `true` and `es_version` is greater than 60 | `7.11.0`. 61 | 62 | See [Doubling down on open, Part II](https://www.elastic.co/blog/licensing-change) 63 | blog post for more details. 64 | 65 | #### How to override configuration files provided by ansible-elasticsearch? 66 | 67 | You can now override the configuration files with your own versions by using the following Ansible variables: 68 | - `es_config_default: "elasticsearch.j2"`: replace `elasticsearch.j2` by your own template to use a custom `/etc/default/elasticsearch`|`/etc/sysconfig/elasticsearch` configuration file 69 | - `es_config_jvm: "jvm.options.j2"`: replace `jvm.options.j2` by your own template to use a custom `/etc/elasticsearch/jvm.options` configuration file 70 | - `es_config_log4j2: ""`: set this variable to the path of your own template to use a custom `/etc/elasticsearch/log4j2.properties` configuration file 71 | 72 | ## Dependency 73 | 74 | This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. 75 | 76 | ## Usage 77 | 78 | Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook. 79 | 80 | ```sh 81 | ansible-galaxy install elastic.elasticsearch,v7.17.0 82 | ``` 83 | 84 | Then create your playbook yaml adding the role elasticsearch. 85 | The application of the elasticsearch role results in the installation of a node on a host. 86 | 87 | The simplest configuration therefore consists of: 88 | 89 | ```yaml 90 | - name: Simple Example 91 | hosts: localhost 92 | roles: 93 | - role: elastic.elasticsearch 94 | vars: 95 | es_version: 7.17.0 96 | ``` 97 | 98 | The above installs Elasticsearch 7.17.0 in a single node 'node1' on the hosts 'localhost'. 99 | 100 | **Note**: 101 | Elasticsearch default version is described in [`es_version`](https://github.com/elastic/ansible-elasticsearch/blob/main/defaults/main.yml#L2). You can override this variable in your playbook to install another version. 102 | While we are testing this role only with one 7.x and one 6.x version (respectively [7.17.0](https://github.com/elastic/ansible-elasticsearch/blob/main/defaults/main.yml#L2) and [6.8.23](https://github.com/elastic/ansible-elasticsearch/blob/main/.kitchen.yml#L22) at the time of writing), this role should work with other versions also in most cases. 103 | 104 | This role also uses [Ansible tags](https://docs.ansible.com/ansible/2.9/user_guide/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information. 105 | 106 | ## Testing 107 | 108 | This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing. 109 | 110 | ### Requirements 111 | 112 | * Ruby 113 | * Bundler 114 | * Docker 115 | * Make 116 | 117 | ### Running the tests 118 | 119 | * Ensure you have checked out this repository to `elasticsearch`, not `ansible-elasticsearch`. 120 | * If you don't have a Gold or Platinum license to test with you can run the trial versions of the `xpack-upgrade` suites by appending `-trial` to the `PATTERN` variable. 121 | * You may need to explicitly specify `VERSION=7.x` if some suites are failing. 122 | 123 | Install the ruby dependencies with bundler 124 | 125 | ```sh 126 | make setup 127 | ``` 128 | 129 | If you want to test X-Pack features with a license you will first need to export the `ES_XPACK_LICENSE_FILE` variable. 130 | ```sh 131 | export ES_XPACK_LICENSE_FILE="$(pwd)/license.json" 132 | ``` 133 | 134 | To converge an Ubuntu 16.04 host running X-Pack 135 | ```sh 136 | $ make converge 137 | ``` 138 | 139 | To run the tests 140 | ```sh 141 | $ make verify 142 | ``` 143 | 144 | To list all of the different test suits 145 | ```sh 146 | $ make list 147 | ``` 148 | 149 | The default test suite is Ubuntu 16.04 with X-Pack. If you want to test another suite you can override this with the `PATTERN` variable 150 | ```sh 151 | $ make converge PATTERN=security-centos-7 152 | ``` 153 | 154 | The `PATTERN` is a kitchen pattern which can match multiple suites. To run all tests for CentOS 155 | ```sh 156 | $ make converge PATTERN=centos-7 157 | ``` 158 | 159 | The default version is 7.x. If you want to test 6.x you can override it with the `VERSION` variable, for example: 160 | ```sh 161 | $ make converge VERSION=6.x PATTERN=security-centos-7 162 | ``` 163 | 164 | When you are finished testing you can clean up everything with 165 | ```sh 166 | $ make destroy-all 167 | ``` 168 | 169 | ### Basic Elasticsearch Configuration 170 | 171 | All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file. 172 | The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters. 173 | 174 | In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. 175 | 176 | The following illustrates applying configuration parameters to an Elasticsearch instance. 177 | 178 | ```yaml 179 | - name: Elasticsearch with custom configuration 180 | hosts: localhost 181 | roles: 182 | - role: elastic.elasticsearch 183 | vars: 184 | es_data_dirs: 185 | - "/opt/elasticsearch/data" 186 | es_log_dir: "/opt/elasticsearch/logs" 187 | es_config: 188 | node.name: "node1" 189 | cluster.name: "custom-cluster" 190 | discovery.seed_hosts: "localhost:9301" 191 | http.port: 9201 192 | transport.port: 9301 193 | node.data: false 194 | node.master: true 195 | bootstrap.memory_lock: true 196 | es_heap_size: 1g 197 | es_api_port: 9201 198 | ``` 199 | 200 | Whilst the role installs Elasticsearch with the default configuration parameters, the following should be configured to ensure a cluster successfully forms: 201 | 202 | * ```es_config['http.port']``` - the http port for the node 203 | * ```es_config['transport.port']``` - the transport port for the node 204 | * ```es_config['discovery.seed_hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) 205 | * ```es_config['cluster.initial_master_nodes']``` - for 7.x and above the list of master-eligible nodes to boostrap the cluster, in the comma separated format ```":,:"``` (typically the node names of the clusters dedicated masters) 206 | * ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. 207 | 208 | The `network.publish_host` setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. 209 | 210 | See https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html for further details on default binding behavior and available options. 211 | The role makes no attempt to enforce the setting of these are requires users to specify them appropriately. It is recommended master nodes are listed and thus deployed first where possible. 212 | 213 | A more complex example: 214 | 215 | ```yaml 216 | - name: Elasticsearch with custom configuration 217 | hosts: localhost 218 | roles: 219 | - role: elastic.elasticsearch 220 | vars: 221 | es_data_dirs: 222 | - "/opt/elasticsearch/data" 223 | es_log_dir: "/opt/elasticsearch/logs" 224 | es_config: 225 | node.name: "node1" 226 | cluster.name: "custom-cluster" 227 | discovery.seed_hosts: "localhost:9301" 228 | http.port: 9201 229 | transport.port: 9301 230 | node.data: false 231 | node.master: true 232 | bootstrap.memory_lock: true 233 | es_heap_size: 1g 234 | es_start_service: false 235 | es_api_port: 9201 236 | es_plugins: 237 | - plugin: ingest-attachment 238 | ``` 239 | 240 | #### Important Notes 241 | 242 | **The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE. These default to "localhost" and 9200 respectively. 243 | If the node is deployed to bind on either a different host or port, these must be changed.** 244 | 245 | **Only use es_data_dirs and es_log_dir for customizing the data and log dirs respectively. When using together with `es_config['path.data']` and `es_config['path.logs']` it would result in generating duplicate data- and logs-keys in `elasticsearch.yml` and thus let fail to start elasticsearch.** 246 | 247 | ### Multi Node Server Installations 248 | 249 | The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. 250 | 251 | An example of a three server deployment is shown below. The first server holds the master and is thus declared first. Whilst not mandatory, this is recommended in any multi node cluster configuration. The two others servers hosts data nodes. 252 | 253 | **Note that we do not support anymore installation of more than one node in the same host** 254 | 255 | ```yaml 256 | - hosts: master_node 257 | roles: 258 | - role: elastic.elasticsearch 259 | vars: 260 | es_heap_size: "1g" 261 | es_config: 262 | cluster.name: "test-cluster" 263 | cluster.initial_master_nodes: "elastic02" 264 | discovery.seed_hosts: "elastic02:9300" 265 | http.host: 0.0.0.0 266 | http.port: 9200 267 | node.data: false 268 | node.master: true 269 | transport.host: 0.0.0.0 270 | transport.port: 9300 271 | bootstrap.memory_lock: false 272 | es_plugins: 273 | - plugin: ingest-attachment 274 | 275 | - hosts: data_node_1 276 | roles: 277 | - role: elastic.elasticsearch 278 | vars: 279 | es_data_dirs: 280 | - "/opt/elasticsearch" 281 | es_config: 282 | cluster.name: "test-cluster" 283 | cluster.initial_master_nodes: "elastic02" 284 | discovery.seed_hosts: "elastic02:9300" 285 | http.host: 0.0.0.0 286 | http.port: 9200 287 | node.data: true 288 | node.master: false 289 | transport.host: 0.0.0.0 290 | transport.port: 9300 291 | bootstrap.memory_lock: false 292 | es_plugins: 293 | - plugin: ingest-attachment 294 | 295 | - hosts: data_node_2 296 | roles: 297 | - role: elastic.elasticsearch 298 | vars: 299 | es_config: 300 | cluster.name: "test-cluster" 301 | discovery.seed_hosts: "elastic02:9300" 302 | http.host: 0.0.0.0 303 | http.port: 9200 304 | node.data: true 305 | node.master: false 306 | transport.host: 0.0.0.0 307 | transport.port: 9300 308 | bootstrap.memory_lock: false 309 | es_plugins: 310 | - plugin: ingest-attachment 311 | ``` 312 | 313 | Parameters can additionally be assigned to hosts using the inventory file if desired. 314 | 315 | Make sure your hosts are defined in your ```inventory``` file with the appropriate ```ansible_ssh_host```, ```ansible_ssh_user``` and ```ansible_ssh_private_key_file``` values. 316 | 317 | Then run it: 318 | 319 | ```sh 320 | ansible-playbook -i hosts ./your-playbook.yml 321 | ``` 322 | 323 | ### Installing X-Pack Features 324 | 325 | * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html) 326 | 327 | 328 | ```yaml 329 | es_role_mapping: 330 | power_user: 331 | - "cn=admins,dc=example,dc=com" 332 | user: 333 | - "cn=users,dc=example,dc=com" 334 | - "cn=admins,dc=example,dc=com" 335 | ``` 336 | 337 | * ```es_users``` - Users can be declared here as yml. Two sub keys 'native' and 'file' determine the realm under which the user is created. Beneath each of these keys users should be declared as yml entries. e.g. 338 | 339 | ```yaml 340 | es_users: 341 | native: 342 | kibana4_server: 343 | password: changeMe 344 | roles: 345 | - kibana4_server 346 | file: 347 | es_admin: 348 | password: changeMe 349 | roles: 350 | - admin 351 | testUser: 352 | password: changeMeAlso! 353 | roles: 354 | - power_user 355 | - user 356 | ``` 357 | 358 | 359 | * ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call. Beneath each key list the roles with appropriate permissions, using the file based format described [here](https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g. 360 | 361 | ```yaml 362 | es_roles: 363 | file: 364 | admin: 365 | cluster: 366 | - all 367 | indices: 368 | - names: '*' 369 | privileges: 370 | - all 371 | power_user: 372 | cluster: 373 | - monitor 374 | indices: 375 | - names: '*' 376 | privileges: 377 | - all 378 | user: 379 | indices: 380 | - names: '*' 381 | privileges: 382 | - read 383 | kibana4_server: 384 | cluster: 385 | - monitor 386 | indices: 387 | - names: '.kibana' 388 | privileges: 389 | - all 390 | native: 391 | logstash: 392 | cluster: 393 | - manage_index_templates 394 | indices: 395 | - names: 'logstash-*' 396 | privileges: 397 | - write 398 | - delete 399 | - create_index 400 | ``` 401 | 402 | * ```es_xpack_license``` - X-Pack license. The license is a json blob. Set the variable directly (possibly protected by Ansible vault) or from a file in the Ansible project on the control machine via a lookup: 403 | 404 | ```yaml 405 | es_xpack_license: "{{ lookup('file', playbook_dir + '/files/' + es_cluster_name + '/license.json') }}" 406 | ``` 407 | 408 | If you don't have a license you can enable the 30-day trial by setting `es_xpack_trial` to `true`. 409 | 410 | X-Pack configuration parameters can be added to the elasticsearch.yml file using the normal `es_config` parameter. 411 | 412 | For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/main/test/integration/xpack-upgrade.yml) 413 | 414 | #### Important Note for Native Realm Configuration 415 | 416 | In order for native users and roles to be configured, the role calls the Elasticsearch API. Given security is installed this requires definition of two parameters: 417 | 418 | * ```es_api_basic_auth_username``` - admin username 419 | * ```es_api_basic_auth_password``` - admin password 420 | 421 | These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme). 422 | 423 | #### X-Pack Security SSL/TLS 424 | 425 | * To configure your cluster with SSL/TLS for HTTP and/or transport communications follow the [SSL/TLS setup procedure](https://github.com/elastic/ansible-elasticsearch/blob/main/docs/ssl-tls-setup.md) 426 | 427 | 428 | ### Additional Configuration 429 | 430 | In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behavior. Options include: 431 | 432 | * ```oss_version``` Default `false`. Setting this to `true` will install the oss release of Elasticsearch (for version <7.11.0 only). 433 | * `es_xpack_trial` Default `false`. Setting this to `true` will start the 30-day trail once the cluster starts. 434 | * ```es_version``` (e.g. "7.17.0"). 435 | * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". 436 | * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** 437 | * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. 438 | * ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username` 439 | * `es_delete_unmanaged_file` Default `true`. Set to false to keep file realm users that have been added outside of ansible. 440 | * `es_delete_unmanaged_native` Default `true`. Set to false to keep native realm users that have been added outside of ansible. 441 | * ```es_start_service``` (true (default) or false) 442 | * ```es_plugins_reinstall``` (true or false (default) ) 443 | * ```es_plugins``` an array of plugin definitions e.g.: 444 | 445 | ```yaml 446 | es_plugins: 447 | - plugin: ingest-attachment 448 | ``` 449 | 450 | * ```es_path_repo``` Sets the whitelist for allowing local back-up repositories 451 | * ```es_action_auto_create_index``` Sets the value for auto index creation, use the syntax below for specifying indexes (else true/false): 452 | es_action_auto_create_index: '[".watches", ".triggered_watches", ".watcher-history-*"]' 453 | * ```es_allow_downgrades``` For development purposes only. (true or false (default) ) 454 | * ```es_java_install``` If set to true, Java will be installed. (false (default for 7.x) or true (default for 6.x)) 455 | * ```update_java``` Updates Java to the latest version. (true or false (default)) 456 | * ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144. 457 | * ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. 458 | * ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds. 459 | * ```es_use_repository``` Setting this to `false` will stop Ansible from using the official Elastic package from any repository configured on the system. 460 | * ```es_add_repository``` Setting this to `false` will stop Ansible to add the official Elastic package repositories (if es_use_repository is true) if you want to use a repo already present. 461 | * ```es_custom_package_url``` the URL to the rpm or deb package for Ansible to install. When using this you will also need to set `es_use_repository: false` and make sure that the `es_version` matches the version being installed from your custom URL. E.g. `es_custom_package_url: https://downloads.example.com/elasticsearch.rpm` 462 | 463 | Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full url. This approach should NOT be used for the X-Pack plugin. See X-Pack below for details here. 464 | 465 | If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration currently has limited support, but more support is planned for later versions. 466 | 467 | To configure X-pack to send mail, the following configuration can be added to the role. When require_auth is true, you will also need to provide the user and password. If not these can be removed: 468 | 469 | ```yaml 470 | es_mail_config: 471 | account: 472 | profile: standard 473 | from: 474 | require_auth: 475 | host: 476 | port: 477 | user: --optional 478 | pass: --optional 479 | ``` 480 | 481 | * ```es_user``` - defaults to elasticsearch. 482 | * ```es_group``` - defaults to elasticsearch. 483 | * ```es_user_id``` - default is undefined. 484 | * ```es_group_id``` - default is undefined. 485 | 486 | Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. 487 | 488 | * ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. 489 | * ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. 490 | 491 | To add, update or remove elasticsearch.keystore entries, use the following variable: 492 | 493 | ```yaml 494 | # state is optional and defaults to present 495 | es_keystore_entries: 496 | - key: someKeyToAdd 497 | value: someValue 498 | state: present 499 | 500 | - key: someKeyToUpdate 501 | value: newValue 502 | # state: present 503 | force: Yes 504 | 505 | - key: someKeyToDelete 506 | state: absent 507 | ``` 508 | 509 | 510 | 511 | This role ships with sample templates located in the [test/integration/files/templates-7.x](https://github.com/elastic/ansible-elasticsearch/tree/main/test/integration/files/templates-7.x) directory. `es_templates_fileglob` variable is used with the Ansible [with_fileglob](http://docs.ansible.com/ansible/playbooks_loops.html#id4) loop. When setting the globs, be sure to use an absolute path. 512 | 513 | ### Proxy 514 | 515 | To define proxy globally, set the following variables: 516 | 517 | * ```es_proxy_host``` - global proxy host 518 | * ```es_proxy_port``` - global proxy port 519 | 520 | ## Notes 521 | 522 | * The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. 523 | * The playbook relies on the inventory_name of each host to ensure its directories are unique 524 | * KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test the latest version of 7.x and 6.x on all supported platforms. 525 | * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. 526 | * In order to run x-pack tests a license file with security enabled is required. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. A trial license is appropriate and can be used by setting `es_xpack_trial` to `true` 527 | 528 | ## IMPORTANT NOTES RE PLUGIN MANAGEMENT 529 | 530 | * If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behavior is required in ES 6.x. 531 | * If no plugins are listed in the playbook for a node, all currently installed plugins will be removed. 532 | * The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed. Should users wish to re-install plugins they should set es_plugins_reinstall to true. This will cause all currently installed plugins to be removed and those listed to be installed. 533 | 534 | ## Questions on Usage 535 | 536 | We welcome questions on how to use the role. However, in order to keep the GitHub issues list focused on "issues" we ask the community to raise questions at https://discuss.elastic.co/c/elasticsearch. This is monitored by the maintainers. 537 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 7.17.0 4 | 5 | * 7.17.0 as default version. 6 | 7 | 8 | | PR | Author | Title | 9 | | --- | --- | --- | 10 | | [#845](https://github.com/elastic/ansible-elasticsearch/pull/845) | [@jmlrt](https://github.com/jmlrt) | Remove CentOS 8 tests | 11 | 12 | 13 | ## 7.16.3 14 | 15 | * 7.16.3 as default version. 16 | * 6.8.23 as 6.x tested version 17 | 18 | ## 7.16.2 19 | 20 | * 7.16.2 as default version. 21 | * 6.8.22 as 6.x tested version 22 | 23 | ## 7.16.1 24 | 25 | * 7.16.1 as default version. 26 | * 6.8.21 as 6.x tested version 27 | 28 | 29 | | PR | Author | Title | 30 | | --- | --- | --- | 31 | | [#838](https://github.com/elastic/ansible-elasticsearch/pull/838) | [@mgreau](https://github.com/mgreau) | Add warning message about 8.x versions | 32 | | [#830](https://github.com/elastic/ansible-elasticsearch/pull/830) | [@bfontaine](https://github.com/bfontaine) | README: fix link to ES licensing change blog post | 33 | | [#831](https://github.com/elastic/ansible-elasticsearch/pull/831) | [@gaima8](https://github.com/gaima8) | command necessarily causes a change on each invocation breaking idempotency, wait_for accomplishes the same goal without the change | 34 | | [#833](https://github.com/elastic/ansible-elasticsearch/pull/833) | [@Pablohn26](https://github.com/Pablohn26) | Fix broken link | 35 | 36 | 37 | ## 7.16.0 38 | 39 | * 7.16.0 as default version. 40 | 41 | 42 | | PR | Author | Title | 43 | | --- | --- | --- | 44 | | [#835](https://github.com/elastic/ansible-elasticsearch/pull/835) | [@jmlrt](https://github.com/jmlrt) | Remove Ubuntu 14.04 support | 45 | | [#825](https://github.com/elastic/ansible-elasticsearch/pull/825) | [@darxriggs](https://github.com/darxriggs) | Remove duplicate entry from YAML | 46 | | [#828](https://github.com/elastic/ansible-elasticsearch/pull/828) | [@ygel](https://github.com/ygel) | Rename master - main | 47 | 48 | 49 | ## 7.15.1 50 | 51 | * 7.15.1 as default version. 52 | 53 | ## 7.15.0 54 | 55 | * 7.15.0 as default version. 56 | 57 | ## 7.14.0 58 | 59 | * 7.14.0 as default version. 60 | * 6.8.18 as 6.x tested version 61 | 62 | ## 7.13.4 63 | 64 | * 7.13.4 as default version. 65 | 66 | ## 7.13.3 67 | 68 | * 7.13.3 as default version. 69 | * 6.8.17 as 6.x tested version 70 | 71 | 72 | | PR | Author | Title | 73 | | --- | --- | --- | 74 | | [#800](https://github.com/elastic/ansible-elasticsearch/pull/800) | [@chuckmilam](https://github.com/chuckmilam) | Minor correction for readability. | 75 | 76 | 77 | ## 7.13.2 78 | 79 | * 7.13.2 as default version. 80 | 81 | ## 7.13.1 82 | 83 | * 7.13.1 as default version. 84 | 85 | ## 7.13.0 86 | 87 | * 7.13.0 as default version. 88 | * 6.8.16 as 6.x tested version 89 | 90 | 91 | | PR | Author | Title | 92 | | --- | --- | --- | 93 | | [#796](https://github.com/elastic/ansible-elasticsearch/pull/796) | [@jmlrt](https://github.com/jmlrt) | Fails deployment when using tls without security | 94 | | [#745](https://github.com/elastic/ansible-elasticsearch/pull/745) | [@v1v](https://github.com/v1v) | Support ubuntu-20 | 95 | 96 | 97 | ## 7.12.1 98 | 99 | * 7.12.1 as default version. 100 | 101 | 102 | | PR | Author | Title | 103 | | --- | --- | --- | 104 | | [#793](https://github.com/elastic/ansible-elasticsearch/pull/793) | [@jmlrt](https://github.com/jmlrt) | [meta] update ubuntu-1804 during kitchen provisioning | 105 | | [#787](https://github.com/elastic/ansible-elasticsearch/pull/787) | [@tobiashuste](https://github.com/tobiashuste) | Stop plugin install to fail in check mode | 106 | 107 | 108 | ## 7.12.0 109 | 110 | * 7.12.0 as default version. 111 | * 6.8.15 as 6.x tested version 112 | 113 | 114 | | PR | Author | Title | 115 | | --- | --- | --- | 116 | | [#789](https://github.com/elastic/ansible-elasticsearch/pull/789) | [@jmlrt](https://github.com/jmlrt) | Use ES_JAVA_HOME instead of JAVA_HOME | 117 | | [#788](https://github.com/elastic/ansible-elasticsearch/pull/788) | [@jmlrt](https://github.com/jmlrt) | Fix deb and rpm url | 118 | | [#784](https://github.com/elastic/ansible-elasticsearch/pull/784) | [@jmlrt](https://github.com/jmlrt) | [meta] fix changelog after 7.11.2 release | 119 | 120 | 121 | ## 7.11.2 122 | 123 | * 7.11.2 as default version. 124 | 125 | | PR | Author | Title | 126 | | --- | --- | --- | 127 | | [#771](https://github.com/elastic/ansible-elasticsearch/pull/771) | [@Bernhard-Fluehmann](https://github.com/Bernhard-Fluehmann) | Cleanup remove keystore entries | 128 | | [#782](https://github.com/elastic/ansible-elasticsearch/pull/782) | [@kuops](https://github.com/kuops) | Fix README.md Multi Node Server Installations | 129 | | [#777](https://github.com/elastic/ansible-elasticsearch/pull/777) | [@DanRoscigno](https://github.com/DanRoscigno) | Update ssl-tls-setup.md | 130 | 131 | 132 | ## 7.11.1 133 | 134 | * 7.11.1 as default version. 135 | * 6.8.14 as 6.x tested version 136 | 137 | | PR | Author | Title | 138 | |-------------------------------------------------------------------|--------------------------------------------------------------|------------------------------------------------| 139 | | [#760](https://github.com/elastic/ansible-elasticsearch/pull/760) | [@jmlrt](https://github.com/jmlrt) | Add dedicated CI jobs for 6.x | 140 | | [#761](https://github.com/elastic/ansible-elasticsearch/pull/761) | [@rubarclk](https://github.com/rubarclk) | Fix Elasticsearch 7.x deb url | 141 | | [#769](https://github.com/elastic/ansible-elasticsearch/pull/769) | [@Bernhard-Fluehmann](https://github.com/Bernhard-Fluehmann) | Add support for elasticsearch-keystore entries | 142 | | [#765](https://github.com/elastic/ansible-elasticsearch/pull/765) | [@jmlrt](https://github.com/jmlrt) | Refactor Kitchen tests | 143 | | [#770](https://github.com/elastic/ansible-elasticsearch/pull/770) | [@jmlrt](https://github.com/jmlrt) | Remove OSS support for version >= 7.11.0 | 144 | | [#779](https://github.com/elastic/ansible-elasticsearch/pull/779) | [@jmlrt](https://github.com/jmlrt) | Fix "list native roles" task | 145 | 146 | 147 | ## 7.10.2 148 | 149 | * 7.10.2 as default version. 150 | 151 | 152 | | PR | Author | Title | 153 | | --- | --- | --- | 154 | | [#753](https://github.com/elastic/ansible-elasticsearch/pull/753) | [@jmlrt](https://github.com/jmlrt) | Fix java install path when system has multiple java | 155 | 156 | 157 | ## 7.10.1 158 | 159 | * 7.10.1 as default version. 160 | 161 | 162 | | PR | Author | Title | 163 | | --- | --- | --- | 164 | | [#747](https://github.com/elastic/ansible-elasticsearch/pull/747) | [@fourstepper](https://github.com/fourstepper) | Fix idempotency for both supported CentOS versions | 165 | | [#744](https://github.com/elastic/ansible-elasticsearch/pull/744) | [@v1v](https://github.com/v1v) | Support CentOS 8 | 166 | | [#736](https://github.com/elastic/ansible-elasticsearch/pull/736) | [@jmlrt](https://github.com/jmlrt) | Fix test-kitchen net-scp-error | 167 | 168 | 169 | ## 7.10.0 170 | 171 | * 7.10.0 as default version. 172 | 173 | 174 | | PR | Author | Title | 175 | | --- | --- | --- | 176 | | [#742](https://github.com/elastic/ansible-elasticsearch/pull/742) | [@jmlrt](https://github.com/jmlrt) | convert custom filter to python3 | 177 | | [#741](https://github.com/elastic/ansible-elasticsearch/pull/741) | [@jmlrt](https://github.com/jmlrt) | [meta] clean deprecated bumper script | 178 | | [#740](https://github.com/elastic/ansible-elasticsearch/pull/740) | [@jmlrt](https://github.com/jmlrt) | fix some typos | 179 | | [#728](https://github.com/elastic/ansible-elasticsearch/pull/728) | [@smutel](https://github.com/smutel) | Improve the documentation for TLS | 180 | | [#739](https://github.com/elastic/ansible-elasticsearch/pull/739) | [@0xflotus](https://github.com/0xflotus) | fix: small error | 181 | 182 | 183 | ## 7.9.3 184 | 185 | * 7.9.3 as default version. 186 | * 6.8.13 as 6.x tested version 187 | 188 | | PR | Author | Title | 189 | | --- | --- | --- | 190 | | [#727](https://github.com/elastic/ansible-elasticsearch/pull/727) | [@smutel](https://github.com/smutel) | Add an option to not upload SSL/TLS certs | 191 | | [#726](https://github.com/elastic/ansible-elasticsearch/pull/726) | [@vielfarbig](https://github.com/vielfarbig) | Add note to only using es_data_dirs and es_log_dir for customizing th… | 192 | 193 | 194 | ## 7.9.2 - 2020/09/24 195 | 196 | * 7.9.2 as default version 197 | 198 | | PR | Author | Title | 199 | |-------------------------------------------------------------------|----------------------------------------|------------------------------------------| 200 | | [#716](https://github.com/elastic/ansible-elasticsearch/pull/716) | [@lksnyder0](https://github.com/lksnyder0) | Use run_once for api related tasks | 201 | 202 | ## 7.9.1 - 2020/09/03 203 | 204 | * 7.9.1 as default version 205 | 206 | | PR | Author | Title | 207 | |-------------------------------------------------------------------|----------------------------------------|------------------------------------------| 208 | | [#701](https://github.com/elastic/ansible-elasticsearch/pull/701) | [@suramon](https://github.com/suramon) | Fix running ansible in check mode | 209 | | [#703](https://github.com/elastic/ansible-elasticsearch/pull/703) | [@anisf](https://github.com/anisf) | Add amazonlinux2 support | 210 | | [#705](https://github.com/elastic/ansible-elasticsearch/pull/705) | [@andzs](https://github.com/andzs) | Use sudo for users migration from <6.3.0 | 211 | 212 | ## 7.9.0 - 2020/08/18 213 | 214 | * 7.9.0 as default version 215 | * 6.8.12 as 6.x tested version 216 | 217 | 218 | ## 7.8.1 - 2020/07/28 219 | 220 | * 7.8.1 as default version 221 | * 6.8.11 as 6.x tested version 222 | 223 | | PR | Author | Title | 224 | |-------------------------------------------------------------------|----------------------------------------|------------------------------------------| 225 | | [#701](https://github.com/elastic/ansible-elasticsearch/pull/701) | [@suramon](https://github.com/suramon) | Fix running ansible in check mode | 226 | | [#703](https://github.com/elastic/ansible-elasticsearch/pull/703) | [@anisf](https://github.com/anisf) | Add amazonlinux2 support | 227 | | [#705](https://github.com/elastic/ansible-elasticsearch/pull/705) | [@andzs](https://github.com/andzs) | Use sudo for users migration from <6.3.0 | 228 | 229 | 230 | ## 7.8.0 - 2020/06/18 231 | 232 | * 7.8.0 as default version 233 | 234 | | PR | Author | Title | 235 | |-------------------------------------------------------------------|------------------------------------|--------------------------------| 236 | | [#653](https://github.com/elastic/ansible-elasticsearch/pull/653) | [@jmlrt](https://github.com/jmlrt) | Fix Xpack features refactoring | 237 | | [#699](https://github.com/elastic/ansible-elasticsearch/pull/699) | [@jmlrt](https://github.com/jmlrt) | Add Debian 10 support | 238 | 239 | ## 7.7.1 - 2020/06/04 240 | 241 | * 7.7.1 as default version 242 | * 6.8.10 as 6.x tested version 243 | 244 | | PR | Author | Title | 245 | |-------------------------------------------------------------------|----------------------------------------------------|----------------------------------| 246 | | [#693](https://github.com/elastic/ansible-elasticsearch/pull/693) | [@jurim76](https://github.com/jurim76) | Fix typo | 247 | | [#697](https://github.com/elastic/ansible-elasticsearch/pull/697) | [@ballesterosam](https://github.com/ballesterosam) | Support limitnofile with systemd | 248 | 249 | ## 7.7.0 - 2020/05/13 250 | 251 | * 7.7.0 as default version 252 | * 6.8.9 as 6.x tested version 253 | * Updated Ansible minimal version from 2.4.2 to 2.5.0 in [#690](https://github.com/elastic/ansible-elasticsearch/pull/690) 254 | 255 | | PR | Author | Title | 256 | |-------------------------------------------------------------------|--------------------------------------------------------|--------------------------------------------------| 257 | | [#689](https://github.com/elastic/ansible-elasticsearch/pull/689) | [@CristianPupazan](https://github.com/CristianPupazan) | Remove port from `initial_master_nodes` setting | 258 | | [#681](https://github.com/elastic/ansible-elasticsearch/pull/691) | [@jmlrt](https://github.com/jmlrt) | Update jvm options with default values for 7.6.0 | 259 | 260 | 261 | ## 7.6.2 - 2020/03/31 262 | 263 | * 7.6.2 as default version 264 | * 6.8.8 as 6.x tested version 265 | 266 | | PR | Author | Title | 267 | |-------------------------------------------------------------------|--------------------------------------|----------------------------------------------------------------| 268 | | [#678](https://github.com/elastic/ansible-elasticsearch/pull/678) | [@nduytg](https://github.com/nduytg) | Update security task files | 269 | | [#681](https://github.com/elastic/ansible-elasticsearch/pull/681) | [@timdev](https://github.com/timdev) | Introduce `es_java_home` variable to allow setting `JAVA_HOME` | 270 | | [#682](https://github.com/elastic/ansible-elasticsearch/pull/682) | [@jmlrt](https://github.com/jmlrt) | Export `VERSION` variable to make subshell | 271 | 272 | 273 | ## 7.6.1 - 2020/03/04 274 | 275 | * 7.6.1 as default version 276 | 277 | | PR | Author | Title | 278 | |-------------------------------------------------------------------|--------------------------------------------------|------------------------------| 279 | | [#674](https://github.com/elastic/ansible-elasticsearch/pull/674) | [@HadrienPatte](https://github.com/HadrienPatte) | Fix typos in README | 280 | | [#672](https://github.com/elastic/ansible-elasticsearch/pull/672) | [@pgassmann](https://github.com/pgassmann) | Fix check mode | 281 | | [#676](https://github.com/elastic/ansible-elasticsearch/pull/676) | [@nduytg](https://github.com/nduytg) | Lint elasticsearch-xpack.yml | 282 | 283 | 284 | ## 7.6.0 - 2020/02/11 285 | 286 | * 7.6.0 as default version 287 | 288 | | PR | Author | Title | 289 | |-------------------------------------------------------------------|--------------------------------------------------------|-------------------------------------------------------| 290 | | [#667](https://github.com/elastic/ansible-elasticsearch/pull/667) | [@dependabot[bot]](https://github.com/apps/dependabot) | Bump rubyzip from 1.2.2 to 2.0.0 | 291 | | [#671](https://github.com/elastic/ansible-elasticsearch/pull/671) | [@haslersn](https://github.com/haslersn) | Remove whitespaces before newmines | 292 | | [#669](https://github.com/elastic/ansible-elasticsearch/pull/669) | [@rs-garrick](https://github.com/rs-garrick) | Several tasks in elasticsearch-ssl.yml missing become | 293 | 294 | 295 | ## 7.5.2 - 2020/01/21 296 | 297 | * 7.5.2 as default version 298 | 299 | | PR | Author | Title | 300 | |-------------------------------------------------------------------|----------------------------------------|-------------------------------------------------------------------| 301 | | [#648](https://github.com/elastic/ansible-elasticsearch/pull/648) | [@jmlrt](https://github.com/jmlrt) | add proxy options to ES_JAVA_OPTS when defined with es_proxy_host | 302 | | [#657](https://github.com/elastic/ansible-elasticsearch/pull/657) | [@jakommo](https://github.com/jakommo) | switched relative URLs to absolute URLs | 303 | | [#664](https://github.com/elastic/ansible-elasticsearch/pull/664) | [@jmlrt](https://github.com/jmlrt) | bump ruby to 2.5.7 | 304 | 305 | 306 | ## 7.5.1 - 2019/12/18 307 | 308 | * 7.5.1 as default version 309 | * 6.8.6 as 6.x tested version 310 | 311 | | PR | Author | Title | 312 | |-------------------------------------------------------------------|------------------------------------|----------------------------------------------------------| 313 | | [#643](https://github.com/elastic/ansible-elasticsearch/pull/643) | [@jmlrt](https://github.com/jmlrt) | Set templates task to run only if `es_templates` is true | 314 | | [#647](https://github.com/elastic/ansible-elasticsearch/pull/647) | [@jmlrt](https://github.com/jmlrt) | Fix when condition for es_ssl_certificate_authority | 315 | 316 | 317 | ## 7.5.0 - 2019/12/09 318 | 319 | * 7.5.0 as default version 320 | * 6.8.5 as 6.x tested version 321 | 322 | ### Breaking changes 323 | 324 | #### Removing the MAX_THREAD settings 325 | 326 | Ansible-elasticsearch 7.5.0 is removing the option to customize the maximum number of threads the process can start in [#637](https://github.com/elastic/ansible-elasticsearch/pull/637/files#diff-04c6e90faac2675aa89e2176d2eec7d8L408). 327 | We discovered that this option wasn't working anymore since multi-instance support removal in ansible-elasticsearch 7.1.1. 328 | This option will be added back in a following release if it's still relevant regarding latest Elasticsearch evolutions. 329 | 330 | #### Changes about configuration files 331 | 332 | Ansible-elasticsearch 7.5.0 is updating the configuration files provided by this role in [#637](https://github.com/elastic/ansible-elasticsearch/pull/637) which contained some otions deprecated in 6.x and 7.x: 333 | - `/etc/default/elasticsearch`|`/etc/sysconfig/elasticsearch`: the new template reflect the configuration file provided by Elasticsearch >= 6.x, the parameter we removed were already not used in 6.x and 7.x 334 | - `/etc/elasticsearch/jvm.options`: the new template reflect the configuration files provided by Elasticsearch >= 6.x 335 | - `/etc/elasticsearch/log4j2.properties`: 336 | - We removed `log4j2.properties.j2` template from this Ansible role as it was a static file not bringing any customization specific to some ansible variable. 337 | - Deployment of this Ansible role on new servers will get the default `log4j2.properties` provided by Elastisearch without any override. 338 | - **WARNING**: For upgrade scenarios where this file was already managed by previous versions of ansible-elasticsearch, this file will become unmanaged and won't be updated by default. If you wish to update it to 7.5 version, you can retrieve it [here](https://github.com/elastic/elasticsearch/blob/7.5/distribution/src/config/log4j2.properties) and use this file with `es_config_log4j2` Ansible variable (see below). 339 | 340 | ##### How to override configuration files provided by ansible-elasticsearch? 341 | 342 | You can now override the configuration files with your own versions by using the following Ansible variables: 343 | - `es_config_default: "elasticsearch.j2"`: replace `elasticsearch.j2` by your own template to use a custom `/etc/default/elasticsearch`|`/etc/sysconfig/elasticsearch` configuration file 344 | - `es_config_jvm: "jvm.options.j2"`: replace `jvm.options.j2` by your own template to use a custom `/etc/elasticsearch/jvm.options` configuration file 345 | - `es_config_log4j2: ""`: set this variable to the path of your own template to use a custom `/etc/elasticsearch/log4j2.properties` configuration file 346 | 347 | ### SSL/TLS Support 348 | 349 | Ansible-elasticsearch is now supporting SSL/TLS encryption. Please refer to [X-Pack Security SSL/TLS](https://github.com/elastic/ansible-elasticsearch/blob/main/docs/ssl-tls-setup.md) to configure it. 350 | 351 | | PR | Author | Title | 352 | |-------------------------------------------------------------------|------------------------------------------------|----------------------------------------------| 353 | | [#625](https://github.com/elastic/ansible-elasticsearch/pull/625) | [@jmlrt](https://github.com/jmlrt) | Add bumper script | 354 | | [#575](https://github.com/elastic/ansible-elasticsearch/pull/575) | [@flyinggecko](https://github.com/flyinggecko) | Docs: Fix name of elasticsearch ansible role | 355 | | [#629](https://github.com/elastic/ansible-elasticsearch/pull/629) | [@patsevanton](https://github.com/patsevanton) | Add cluster.initial_master_nodes | 356 | | [#620](https://github.com/elastic/ansible-elasticsearch/pull/620) | [@pemontto](https://github.com/pemontto) | Add SSL/TLS support | 357 | | [#630](https://github.com/elastic/ansible-elasticsearch/pull/630) | [@jmlrt](https://github.com/jmlrt) | Indent yaml for config file | 358 | | [#636](https://github.com/elastic/ansible-elasticsearch/pull/636) | [@jmlrt](https://github.com/jmlrt) | Bump elasticsearch to 6.8.5 and 7.4.2 | 359 | | [#637](https://github.com/elastic/ansible-elasticsearch/pull/637) | [@jmlrt](https://github.com/jmlrt) | Use default config files | 360 | 361 | 362 | ## 7.4.1 - 2019/10/23 363 | 364 | * 7.4.1 as default version 365 | * 6.8.4 as 6.x tested version 366 | 367 | | PR | Author | Title | 368 | |-------------------------------------------------------------------|------------------------------------|----------------------------------------------------| 369 | | [#617](https://github.com/elastic/ansible-elasticsearch/pull/617) | [@jmlrt](https://github.com/jmlrt) | Use systemd ansible module for daemon-reload | 370 | | [#618](https://github.com/elastic/ansible-elasticsearch/pull/618) | [@jmlrt](https://github.com/jmlrt) | Fix probot newlines | 371 | | [#619](https://github.com/elastic/ansible-elasticsearch/pull/619) | [@jmlrt](https://github.com/jmlrt) | Fix python AttributeError + format code with black | 372 | 373 | 374 | ## 7.4.0 - 2019/10/01 375 | 376 | * 7.4.0 as default version 377 | * Remove compatibility with versions < 6.3 378 | 379 | | PR | Author | Title | 380 | |-------------------------------------------------------------------|----------------------------------------------------------|--------------------------------------------------------------------------------------| 381 | | [#575](https://github.com/elastic/ansible-elasticsearch/pull/575) | [@flyinggecko](https://github.com/flyinggecko) | Fix name of Elasticsearch Ansible role | 382 | | [#578](https://github.com/elastic/ansible-elasticsearch/pull/578) | [@jmlrt](https://github.com/jmlrt) | Fix `dict object has no attribute dict_keys` issue with Python3 | 383 | | [#588](https://github.com/elastic/ansible-elasticsearch/pull/588) | [@broferek](https://github.com/broferek) | Move `userid` and `groupid` in a different place in the role | 384 | | [#591](https://github.com/elastic/ansible-elasticsearch/pull/591) | [@Crazybus](https://github.com/Crazybus) | Add back in `force_basic_auth` for all http requests | 385 | | [#582](https://github.com/elastic/ansible-elasticsearch/pull/582) | [@ktibi](https://github.com/ktibi) | Allow disable Elastic official repository setup | 386 | | [#593](https://github.com/elastic/ansible-elasticsearch/pull/593) | [@jmlrt](https://github.com/jmlrt) | Bunch of small fixes | 387 | | [#595](https://github.com/elastic/ansible-elasticsearch/pull/595) | [@broferek](https://github.com/broferek) | Set `limitMEMLOCK` for OS using Systemd | 388 | | [#600](https://github.com/elastic/ansible-elasticsearch/pull/600) | [@titan-architrave](https://github.com/titan-architrave) | Always gather the `es_major_version` variables | 389 | | [#605](https://github.com/elastic/ansible-elasticsearch/pull/605) | [@jmlrt](https://github.com/jmlrt) | Add doc for migration with data move | 390 | | [#601](https://github.com/elastic/ansible-elasticsearch/pull/601) | [@LukeRoz](https://github.com/LukeRoz) | Removing package version hold when `es_version_hold: false` | 391 | | [#612](https://github.com/elastic/ansible-elasticsearch/pull/612) | [@jmlrt](https://github.com/jmlrt) | Add Probot config to manage stale issues/pr | 392 | | [#614](https://github.com/elastic/ansible-elasticsearch/pull/614) | [@jmlrt](https://github.com/jmlrt) | Describe how to select a different elasticsearch version | 393 | | [#609](https://github.com/elastic/ansible-elasticsearch/pull/609) | [@jmlrt](https://github.com/jmlrt) | No more 6.3 compatibility + Use default files permissions from Elasticsearch package | 394 | | [#510](https://github.com/elastic/ansible-elasticsearch/pull/510) | [@verboEse](https://github.com/verboEse) | Don't fetch APT key if existent | 395 | 396 | 397 | ## 7.1.1 - 2019/06/04 398 | 399 | ### Breaking changes 400 | 401 | #### End of multi-instance support 402 | 403 | * Starting with ansible-elasticsearch:7.1.1, installing more than one instance of Elasticsearch **on the same host** is no longer supported. 404 | * Configuration, datas, logs and PID directories are now using standard paths like in the official Elasticsearch packages. 405 | 406 | * If you use only one instance but want to upgrade from an older ansible-elasticsearch version, follow [upgrade procedure](./docs/multi-instance.md#upgrade-procedure) 407 | * If you install more than one instance of Elasticsearch on the same host (with different ports, directory and config files), **do not update to ansible-elasticsearch >= 7.1.1**, please follow this [workaround](./docs/multi-instance.md#workaround) instead. 408 | * For multi-instances use cases, we are now recommending Docker containers using our official images (https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html). 409 | 410 | #### Moved some security features to basic 411 | 412 | You can now using basic authentication by overriding `es_api_basic_auth_username` and `es_api_basic_auth_password` variables without providing a license file. 413 | 414 | ### Features 415 | 416 | * 7.1.1 as default Elasticsearch version 417 | * [#539](https://github.com/elastic/ansible-elasticsearch/pull/539) and [#542](https://github.com/elastic/ansible-elasticsearch/pull/542) - @grzegorznowak - Make ansible role compatible with ansible [check mode](https://docs.ansible.com/ansible/latest/user_guide/playbooks_checkmode.html) 418 | * [#558](https://github.com/elastic/ansible-elasticsearch/pull/558) - @jmlrt - Add support for Elasticsearch 7.x, remove 5.x support and update tests 419 | * [#560](https://github.com/elastic/ansible-elasticsearch/pull/560) - @jmlrt - Use default xpack features and remove system_key deprecated feature 420 | * [#562](https://github.com/elastic/ansible-elasticsearch/pull/562) - @hamishforbes - Allow to customize instance suffix 421 | * [#566](https://github.com/elastic/ansible-elasticsearch/pull/566) - @jmlrt - Remove multi-instances support 422 | * [#567](https://github.com/elastic/ansible-elasticsearch/pull/567) - @jmlrt - Remove file scripts deprecated feature 423 | * [#568](https://github.com/elastic/ansible-elasticsearch/pull/568) - @jmlrt - Skip Java install for Elasticsearch 7.x (java is now embeded) 424 | 425 | ### Fixes 426 | 427 | * [#543](https://github.com/elastic/ansible-elasticsearch/pull/543) - @victorgs - Fix typo in Makefile 428 | * [#546](https://github.com/elastic/ansible-elasticsearch/pull/546) - @thiagonache - Fix README example 429 | * [#550](https://github.com/elastic/ansible-elasticsearch/pull/550) - @pemontto - Fix template conditional 430 | * [#556](https://github.com/elastic/ansible-elasticsearch/pull/556) - @jmlrt - Fix debian-8 test 431 | * [#557](https://github.com/elastic/ansible-elasticsearch/pull/557) - @jmlrt - Bump gem dependencies to fix [CVE-2018-1000544](https://nvd.nist.gov/vuln/detail/CVE-2018-1000544) and [CVE-2018-1000201](https://nvd.nist.gov/vuln/detail/CVE-2018-1000201) 432 | * [#564](https://github.com/elastic/ansible-elasticsearch/pull/564) - @jmlrt - Bump all gem dependencies to fix kitchen tests 433 | 434 | 435 | ## 6.6.0 - 2019/01/29 436 | 437 | ### Features 438 | 439 | * 6.6.0 as default Elasticsearch version 440 | * [#521](https://github.com/elastic/ansible-elasticsearch/pull/521) - @Crazybus - Allow switching between oss and standard packages 441 | * [#528](https://github.com/elastic/ansible-elasticsearch/pull/528) - @Fra-nk - Use systemd's RequiresMountsFor 442 | * [#530](https://github.com/elastic/ansible-elasticsearch/pull/530) - @lde - Use dpkg_selections to lock Elasticsearch version 443 | 444 | ### Fixes 445 | 446 | * [#513](https://github.com/elastic/ansible-elasticsearch/pull/513) - @kakoni - Fix typo in elasticsearch-parameters.yml 447 | * [#522](https://github.com/elastic/ansible-elasticsearch/pull/522) - @SlothOfAnarchy - Fix package download URL 448 | * [#526](https://github.com/elastic/ansible-elasticsearch/pull/526) - @Fra-nk - Allow not installing Elasticsearch deb repository key 449 | * [#527](https://github.com/elastic/ansible-elasticsearch/pull/527) - @katsukamaru - Execute java version check in check mode 450 | 451 | 452 | ## 6.5.1.1 - 2018/11/27 453 | 454 | ### Fixes 455 | 456 | * [#516](https://github.com/elastic/ansible-elasticsearch/pull/516) - @Crazybus - Only attempt to copy the old users file if it actually exists 457 | 458 | 459 | ## 6.5.1 - 2018/11/26 460 | 461 | ### Features 462 | 463 | * 6.5.1 as default Elasticsearch version 464 | 465 | ### Fixes 466 | 467 | * [#487](https://github.com/elastic/ansible-elasticsearch/pull/487) - @lazouz - Disable check mode to make install plugins idempotent 468 | * [#501](https://github.com/elastic/ansible-elasticsearch/pull/501) - @kaxil - Make the order of configs consistent for comparing 469 | * [#497](https://github.com/elastic/ansible-elasticsearch/pull/497) - @Crazybus - Document es_use_repository and es_custom_package_url 470 | * [#504](https://github.com/elastic/ansible-elasticsearch/pull/504) - @victorgs - Using tests as filters is deprecated 471 | * [#493](https://github.com/elastic/ansible-elasticsearch/pull/493) - @Crazybus - Only use the first found java version if there are multiple installed 472 | 473 | 474 | ## 6.4.0 - 2018/08/24 475 | 476 | ### Features 477 | 478 | * 6.4.0 as default Elasticsearch version 479 | 480 | ### Fixes 481 | 482 | * [#484](https://github.com/elastic/ansible-elasticsearch/pull/484) - @kimoto - Fix downgrading Elasticsearch on RedHat hosts 483 | * [#476](https://github.com/elastic/ansible-elasticsearch/pull/476) - @Crazybus - Fix version locking for the elasticsearch-oss package 484 | 485 | 486 | ## 6.3.1 - 2018/07/05 487 | 488 | ### Features 489 | 490 | * 6.3.1 as default Elasticsearch version 491 | 492 | 493 | ## 6.3.0.1 - 2018/06/28 494 | 495 | ### Fixes 496 | 497 | * [#460](https://github.com/elastic/ansible-elasticsearch/pull/460) - @toadjaune - Make sure ansible doesn't fail if the default systemd service file doesn't exist 498 | * [#461](https://github.com/elastic/ansible-elasticsearch/pull/461) - @bilsch - Add missing become root in tasks that require root access 499 | 500 | 501 | ## 6.3.0 - 2018/06/18 502 | 503 | ### Breaking changes 504 | 505 | Elasticsearch 6.3 includes several big changes that are reflected in this role. 506 | When upgrading from module versions prior to 6.3, there are a number of upgrade considerations to take into account: 507 | 508 | * This role defaults to the upstream package repositories, which now include X-Pack bundled by default. To preserve previous behavior which does _not_ include X-Pack be sure to explicitly set `es_enable_xpack: false` which will install the `elasticsearch-oss` package. 509 | * Great care has been taken in making sure that all upgrade paths work, however as always please take extra caution when upgrading and test in a non-production environment. New automated tests have been added to make sure that the following upgrade paths work: 510 | * oss to oss 511 | * oss to xpack 512 | * xpack to xpack 513 | * X-Pack configuration files which used to be in `${ES_PATH_CONF}/x-pack` are now in `${ES_PATH_CONF}/`. If you have any configuration files in this directory not managed by ansible you will need to move them manually. 514 | 515 | #### Features 516 | 517 | * Integration testing has been refactored in [#457](https://github.com/elastic/ansible-elasticsearch/pull/457). This removed a lot of duplicate tests and added new tests to make sure all upgrade paths work. 518 | * It is now possible to test Elasticsearch snapshot builds by setting `es_use_snapshot_release` to `true` 519 | 520 | #### Fixes 521 | 522 | * Installing `es_plugins` from custom urls is now idempotent. Previously the plugin name was being compared to the url which meant it would be reinstalled every time ansible was run because they didn't match 523 | 524 | #### Pull requests 525 | 526 | * [#452](https://github.com/elastic/ansible-elasticsearch/pull/452) - @Crazybus - Add initial 6.3 support 527 | * [#454](https://github.com/elastic/ansible-elasticsearch/pull/454) - @Crazybus - Move jenkins matrix file into the repo so test suites are controlled via the pull request workflow 528 | * [#455](https://github.com/elastic/ansible-elasticsearch/pull/455) - @Crazybus - Add automated test for upgrading from oss to oss 529 | * [#457](https://github.com/elastic/ansible-elasticsearch/pull/457) - @Crazybus - Refactor integration tests to remove duplication and add extra suites to make sure all upgrade paths are covered 530 | 531 | 532 | ## 6.2.4.1 - 2018/06/14 533 | 534 | Patch release requested by @average-joe in #453 535 | 536 | #### Pull requests 537 | 538 | * [#445](https://github.com/elastic/ansible-elasticsearch/pull/445) - @gekkeharry13 - Added configuration options for configuring x-pack notifications via email with some other nice fixes. 539 | * [#450](https://github.com/elastic/ansible-elasticsearch/pull/450) - @Crazybus - improving some flakey tests which were randomly failing. 540 | * [#447](https://github.com/elastic/ansible-elasticsearch/pull/447) - @chaintng - Fix to make sure sudo is used when running `update-alternatives` for java. 541 | * [#423](https://github.com/elastic/ansible-elasticsearch/pull/423) - @eRadical - Fixing the until condition being used when installing rpms from a custom repository. 542 | 543 | 544 | ## 6.2.4 - 2018/04/24 545 | 546 | * `6.2.4` and `5.6.9` as the default versions. 547 | 548 | 549 | ## 6.2.3 - 2018/04/21 550 | 551 | * Thanks to @cl0udf0x for adding proper names to all tasks which were unnamed [#417](https://github.com/elastic/ansible-elasticsearch/pull/417) 552 | * Thanks @cyrilleverrier for having a keen eye and spotting this typo. [#432](https://github.com/elastic/ansible-elasticsearch/pull/432) 553 | 554 | 555 | ## 6.2.2 - 2018/02/22 556 | 557 | * `6.2.2` and `5.6.8` as the default versions 558 | * Thanks to @pemontto for fixing up all of the ansible conditional logic https://github.com/elastic/ansible-elasticsearch/pull/429 559 | * Thanks @cyrilleverrier for https://github.com/elastic/ansible-elasticsearch/pull/427 which makes sure x-pack settings are not in the config file when x-pack isn't enabled 560 | 561 | 562 | ## 6.1.3 - 2018/02/01 563 | 564 | * `6.x` is now the default `es_major_version` with `6.1.3` as the default `es_version` 565 | * Special thanks to @shribigb, @toddlers and @remil1000 for their efforts in getting `6.x` support working! 566 | * `.kitchen.yml` has been updated to allow testing both `6.x` and `5.x` versions 567 | * A new [Jenkins job](https://devops-ci.elastic.co/job/elastic+ansible-elasticsearch+pull-request/) has been added for pull requests to automatically test all combinations of `6.x` and `5.x` on ubuntu-1404, ubuntu-1604, debian-8 and centos-7 with the various test suites. 568 | 569 | 570 | ## 5.5.1 - 2017/08/20 571 | 572 | * Fixes with respect to issues on restart. 573 | * 5.5.1 update with supporting package scripts. 574 | * Documentation clarification. 575 | * Fixes for loading of templates 576 | * Support for ML 577 | * Ability to install x-pack from remote. 578 | --------------------------------------------------------------------------------