├── tests
    ├── ansible.cfg
    ├── local.ini
    ├── test1.yml
    ├── elastic_test.sh
    └── test1_var.yml
├── tasks
    ├── shield_config.yml
    ├── shield.yml
    ├── esusers.yml
    ├── timezone.yml
    ├── java.yml
    ├── es_snapshot.yml
    ├── es_restore.yml
    ├── post-run.yml
    ├── marvel.yml
    ├── custom-jars.yml
    ├── aws.yml
    ├── spm.yml
    ├── main.yml
    ├── plugins.yml
    └── elastic-install.yml
├── handlers
    └── main.yml
├── meta
    └── main.yml
├── vagrant-inventory.ini
├── vagrant-main.yml
├── .gitignore
├── setup.sh
├── vars
    ├── sample.yml
    └── vagrant.yml
├── defaults
    └── main.yml
├── Vagrantfile
├── .travis.yml
├── templates
    ├── elasticsearch.default.j2
    ├── elasticsearch.in.sh.j2
    └── elasticsearch.yml.j2
├── library
    ├── es_snapshot_restore.py
    └── es_reindex.py
└── README.md


/tests/ansible.cfg:
--------------------------------------------------------------------------------
1 | [defaults]
2 | roles_path=../../


--------------------------------------------------------------------------------
/tests/local.ini:
--------------------------------------------------------------------------------
1 | localhost ansible_connection='local'


--------------------------------------------------------------------------------
/tasks/shield_config.yml:
--------------------------------------------------------------------------------
1 | - copy: src={{elasticsearch_shield_files}} dest=/etc/elasticsearch/shield/


--------------------------------------------------------------------------------
/tests/test1.yml:
--------------------------------------------------------------------------------
1 | ---
2 | # This playbook nexus
3 | - hosts: all
4 |   sudo: true
5 |   vars_files:
6 |     - "test1_var.yml"
7 |   roles:
8 |     - ansible-elasticsearch
9 | 


--------------------------------------------------------------------------------
/handlers/main.yml:
--------------------------------------------------------------------------------
1 | ---
2 | # Elasticsearch Ansible Handlers
3 | 
4 | # Restart Elasticsearch
5 | - name: Restart Elasticsearch
6 |   service: name=elasticsearch state=restarted
7 | 


--------------------------------------------------------------------------------
/tasks/shield.yml:
--------------------------------------------------------------------------------
1 | - include: esusers.yml
2 |   when: (elasticsearch_esusers is defined)
3 | 
4 | - include: shield_config.yml
5 |   tags: shield_config_files
6 |   when: (elasticsearch_shield_files is defined)
7 |   notify: Restart Elasticsearch
8 | 


--------------------------------------------------------------------------------
/meta/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Elasticsearch Ansible Meta
 3 | galaxy_info:
 4 |   author: "George Stathis"
 5 |   company: Traackr
 6 |   license: MIT
 7 |   min_ansible_version: 1.3
 8 |   platforms:
 9 |    - name: Ubuntu
10 |      versions:
11 |       - precise
12 |   categories:
13 |    - database:nosql
14 | dependencies: []
15 | 


--------------------------------------------------------------------------------
/vagrant-inventory.ini:
--------------------------------------------------------------------------------
 1 | #
 2 | # Inventory for provisioning with Vagrant
 3 | #
 4 | 
 5 | #####################
 6 | # Local Environment #
 7 | #####################
 8 | [vagrant]
 9 | 192.168.111.10 ansible_ssh_user=vagrant ansible_ssh_pass=vagrant ansible_connection=local
10 | 
11 | [vagrant:vars]
12 | # spm_client_token=<enter your token>
13 | 


--------------------------------------------------------------------------------
/tests/elastic_test.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -e
 3 | 
 4 | # Put data
 5 | curl -XPUT 'http://localhost:9200/blog/user/dilbert' -d '{ "name" : "Dilbert Brown" }'
 6 | 
 7 | #Get data
 8 | curl -XGET 'http://localhost:9200/blog/user/dilbert?pretty=true' | grep "\"name\" : \"Dilbert Brown\""
 9 | 
10 | # Check if kopf is running
11 | curl -XGET 'http://localhost:9200/_plugin/kopf/' | grep "ng-app=\"kopf\""
12 | 


--------------------------------------------------------------------------------
/vagrant-main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Elasticsearch Ansible Playbook
 3 | - hosts: all
 4 |   user: $user
 5 |   sudo: yes
 6 | 
 7 |   vars_files:
 8 |     - defaults/main.yml
 9 |     - vars/vagrant.yml
10 | 
11 |   tasks:
12 |     - include: tasks/main.yml
13 |     # Uncomment to install and enable SPM. Make sure to set the spm_client_token variable in your inventory.ini to your SPM key
14 |     # - include: tasks/spm.yml
15 | 
16 |   handlers:
17 |     - include: handlers/main.yml
18 | 


--------------------------------------------------------------------------------
/tasks/esusers.yml:
--------------------------------------------------------------------------------
 1 | - name: Delete user from esuser realm if it exists
 2 |   action: >
 3 |     shell bin/shield/esusers userdel {{ item.username }}
 4 |     chdir={{ elasticsearch_home_dir }}
 5 |   ignore_errors: yes
 6 |   with_items: elasticsearch_esusers
 7 | - name: Add user to esuser realm
 8 |   action: >
 9 |     shell bin/shield/esusers useradd "{{ item.username }}" -p "{{ item.password }}" -r "{{ item.role }}"
10 |     chdir={{ elasticsearch_home_dir }}
11 |   ignore_errors: no
12 |   with_items: elasticsearch_esusers
13 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Numerous always-ignore extensions
 2 | *.diff
 3 | *.err
 4 | *.orig
 5 | *.log
 6 | *.rej
 7 | *.swo
 8 | *.swp
 9 | *.vi
10 | *~
11 | *.sass-cache
12 | *.iml
13 | 
14 | # OS or Editor folders
15 | .DS_Store
16 | Thumbs.db
17 | .cache
18 | .project
19 | .classpath
20 | .settings
21 | .tmproj
22 | *.esproj
23 | nbproject
24 | *.sublime-project
25 | *.sublime-workspace
26 | *.ipr
27 | *.iws
28 | 
29 | # Folders to ignore
30 | .hg
31 | .svn
32 | .CVS
33 | intermediate
34 | publish
35 | .idea
36 | target
37 | target-eclipse
38 | .vagrant
39 | 


--------------------------------------------------------------------------------
/tasks/timezone.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: timezone | check current timezone
 4 |   shell: cat /etc/timezone
 5 |   changed_when: 0
 6 |   register: current_zone
 7 | 
 8 | - name: timezone | Set timezone variables
 9 |   copy:
10 |     content={{elasticsearch_timezone}}
11 |     dest=/etc/timezone
12 |     owner=root
13 |     group=root
14 |     mode=0644
15 |   when: current_zone.stdout != elasticsearch_timezone
16 | 
17 | - name: timezone | Run dpkg-reconfigure to configure timezone
18 |   shell: dpkg-reconfigure --frontend noninteractive tzdata
19 |   when: current_zone.stdout != elasticsearch_timezone


--------------------------------------------------------------------------------
/setup.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | export PYTHONUNBUFFERED=1
 3 | export ANSIBLE_FORCE_COLOR=1
 4 | 
 5 | if [ $(dpkg-query -W -f='${Status}' ansible 2>/dev/null | grep -c "ok installed") -eq 0 ];
 6 | then
 7 |     echo "Add APT repositories"
 8 |     export DEBIAN_FRONTEND=noninteractive
 9 |     apt-get install -qq software-properties-common &> /dev/null || exit 1
10 |     apt-add-repository ppa:ansible/ansible &> /dev/null || exit 1
11 | 
12 |     apt-get update -qq
13 | 
14 |     echo "Installing Ansible"
15 |     apt-get install -qq ansible &> /dev/null || exit 1
16 |     echo "Ansible installed"
17 | fi
18 | 
19 | 


--------------------------------------------------------------------------------
/tasks/java.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: java | Accept Oracle license prior JDK installation
 4 |   shell: echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections; echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
 5 |     creates=/usr/lib/jvm/java-8-oracle
 6 | 
 7 | - name: java | Update repositories
 8 |   apt_repository:
 9 |     repo={{ item }}
10 |     state=present
11 |     update_cache=yes
12 |   with_items: elasticsearch_apt_repos
13 | 
14 | - name: java | Install dependencies
15 |   apt:
16 |     pkg={{elasticsearch_apt_java_package}}
17 |     state=present
18 | 


--------------------------------------------------------------------------------
/tasks/es_snapshot.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 |  - name: check backup folder exists.
 3 |    sudo: true
 4 |    file: path="{{elasticsearch_repo_dir}}" state=directory owner={{elasticsearch_user}} group={{elasticsearch_group}} recurse=yes
 5 | 
 6 |  - name: install pip
 7 |    sudo: true
 8 |    apt: name=python-pip state=present
 9 | 
10 |  - name: install elasticsearch pip
11 |    pip: name=elasticsearch state=present
12 | 
13 |  - name: check elasticsearch repository directory exists
14 |    file: path={{elasticsearch_repo_dir}} state=directory owner={{elasticsearch_user}}
15 | 
16 |  - name: create es backups
17 |    es_snapshot_restore: elasticsearch_host={{elasticsearch_backup_host}} repository_path="{{elasticsearch_repo_dir}}" repository_name={{elasticsearch_repo_name }}  snapshot_name="{{elasticsearch_snapshot_name}}" snapshot_indices="{{elasticsearch_snapshot_indices | default(omit) }}" mode=snapshot state=present
18 | 


--------------------------------------------------------------------------------
/tasks/es_restore.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 |  - name: copy in backup folder from somewhere.
 3 |    sudo: true
 4 |    synchronize: src={{backup_source}} dest="{{elasticsearch_repo_dir}}"
 5 | 
 6 |  - name: install pip
 7 |    sudo: true
 8 |    apt: name=python-pip state=present
 9 | 
10 |  - name: install elasticsearch pip
11 |    pip: name=elasticsearch state=present
12 | 
13 |  - name: set es template  mapping if provided
14 |    shell: curl -sS -XPUT '{{elasticsearch_backup_host}}/_template/{{snapshot_indices}}' --data @{{snapshot_template_files }}
15 |    when: snapshot_template_files is defined
16 | 
17 | 
18 |  - name: restore es wiki backups
19 |    es_snapshot_restore: elasticsearch_host={{elasticsearch_backup_host}} repository_path="{{elasticsearch_repo_dir}}" repository_name={{elasticsearch_repo_name }} snapshot_name="{{elasticsearch_snapshot_restore_name}}" snapshot_indices="{{elasticsearch_snapshot_indices | default(omit) }}" mode=restore state=present
20 | 


--------------------------------------------------------------------------------
/tests/test1_var.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | elasticsearch_version: 1.4.2
 4 | elasticsearch_apt_java_package: oracle-java8-installer
 5 | elasticsearch_java_home: /usr/lib/jvm/java-8-oracle
 6 | elasticsearch_heap_size: 1g
 7 | elasticsearch_max_open_files: 65535
 8 | elasticsearch_timezone: "America/New_York"
 9 | elasticsearch_node_max_local_storage_nodes: 1
10 | elasticsearch_index_mapper_dynamic: "true"
11 | elasticsearch_memory_bootstrap_mlockall: "true"
12 | elasticsearch_install_java: "true"
13 | elasticsearch_plugins:
14 |   - { name: 'elasticsearch/elasticsearch-mapper-attachments/2.4.1' }
15 |   - { name: 'com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/2.0.5' }
16 |   - { name: 'facet-script', url: 'http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-facet-script-1.1.2.zip' }
17 |   - { name: 'lmenezes/elasticsearch-kopf' }
18 | elasticsearch_thread_pools:
19 |   - "threadpool.bulk.type: fixed"
20 |   - "threadpool.bulk.size: 50"
21 |   - "threadpool.bulk.queue_size: 1000"
22 | 


--------------------------------------------------------------------------------
/tasks/post-run.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: post-run | Ensure plugins directory permission are correct
 4 |   file:
 5 |     path="{{ elasticsearch_plugin_dir }}"
 6 |     state=directory
 7 |     owner={{ elasticsearch_user }}
 8 |     group={{ elasticsearch_group }}
 9 |     recurse=yes
10 | 
11 | - name: post-run | Ensure Elasticsearch is running and started on boot
12 |   service:
13 |     name=elasticsearch
14 |     enabled={{ elasticsearch_service_startonboot }}
15 |     state={{ elasticsearch_service_state }}
16 |   when: installed_version.stat.exists
17 | 
18 | # Flush handlers to restart elastic if needed
19 | - meta: flush_handlers
20 | 
21 | - name: post-run | Check http port is open and running version. timeout 160s
22 |   wait_for:
23 |     host={{ '127.0.0.1' if elasticsearch_network_bind_host is not defined or elasticsearch_network_bind_host == '0.0.0.0' else elasticsearch_network_bind_host }}
24 |     port={{ elasticsearch_network_http_port | default('9200') }}
25 |     timeout=160
26 |   when: elasticsearch_service_state == "started"


--------------------------------------------------------------------------------
/vars/sample.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Elasticsearch Ansible Sample Variables
 3 | 
 4 | elasticsearch_version: 0.90.5
 5 | elasticsearch_heap_size: 1g
 6 | elasticsearch_max_open_files: 65535
 7 | elasticsearch_max_locked_memory: unlimited
 8 | elasticsearch_timezone: "America/New_York"
 9 | elasticsearch_cluster_name: elasticsearch-ansible
10 | elasticsearch_node_name: elasticsearch-ansible-node
11 | elasticsearch_node_max_local_storage_nodes: 1
12 | elasticsearch_index_mapper_dynamic: "true"
13 | elasticsearch_memory_bootstrap_mlockall: "true"
14 | elasticsearch_gateway_type: local
15 | elasticsearch_gateway_recover_after_nodes: 1
16 | elasticsearch_gateway_recover_after_time: 2m
17 | elasticsearch_gateway_expected_nodes: 1
18 | elasticsearch_discovery_zen_minimum_master_nodes: 1
19 | elasticsearch_discovery_zen_ping_timeout: 30s
20 | elasticsearch_discovery_zen_ping_multicast_enabled: "true"
21 | elasticsearch_misc_auto_create_index: "true"
22 | elasticsearch_misc_query_bool_max_clause_count: 4096
23 | elasticsearch_misc_disable_delete_all_indices: "true"
24 | elasticsearch_java_opts: "-XX:-UseSuperWord"
25 | 


--------------------------------------------------------------------------------
/tasks/marvel.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Install Marvel (see http://www.elasticsearch.org/guide/en/marvel/current/)
 3 | #
 4 | # The following variables need to be defined in your playbook or inventory:
 5 | # - elasticsearch_plugin_marvel_version
 6 | #
 7 | # The following variables provide configuration for the plugin.
 8 | # More options may be available in the future:
 9 | # - elasticsearch_plugin_marvel_agent_enabled
10 | # - elasticsearch_plugin_marvel_agent_exporter_es_hosts
11 | # - elasticsearch_plugin_marvel_agent_indices
12 | # - elasticsearch_plugin_marvel_agent_interval
13 | # - elasticsearch_plugin_marvel_agent_exporter_es_index_timeformat
14 | 
15 | - name: marvel | Removing Marvel Plugin if it exists
16 |   shell: bin/plugin --remove marvel
17 |     chdir={{ elasticsearch_home_dir }}
18 |   ignore_errors: yes
19 | - name: marvel | Installing Marvel Plugin
20 |   shell: bin/plugin -i elasticsearch/marvel/{{ elasticsearch_plugin_marvel_version }}
21 |     chdir={{ elasticsearch_home_dir }}
22 |   register: marvel_plugins_installed
23 |   changed_when: "'Installed' in marvel_plugins_installed.stdout"
24 |   failed_when: "marvel_plugins_installed.rc != 0 and marvel_plugins_installed.stdout.find('already exists. To update the plugin') == -1"
25 | 


--------------------------------------------------------------------------------
/tasks/custom-jars.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Install Custom JARs
 3 | #
 4 | # Custom jars are made available to the Elasticsearch classpath by being downloaded into the elasticsearch_home_dir/lib folder.
 5 | # An example of a custom jar can include a custom Lucene Similarity Provider. You will need to define an array called
 6 | # 'elasticsearch_custom_jars' in your playbook or inventory, such that:
 7 | #
 8 | # elasticsearch_custom_jars:
 9 | #  - { uri: '<URL where JAR can be downloaded from: required>', filename: '<alternative name for final JAR if different from file downladed: leave blank to use same filename>', user: '<BASIC auth username: leave blank of not needed>', passwd: '<BASIC auth password: leave blank of not needed>' }
10 | #  - ...
11 | 
12 | # Loop though elasticsearch_custom_jars and install them
13 | - name: Installing Custom JARs
14 |   action: >
15 |     get_url url={{ item.uri }}
16 |     url_username={{ item.user }} url_password={{ item.passwd }} dest="{{ elasticsearch_home_dir }}/lib/{{ item.filename }}"
17 |   with_items: elasticsearch_custom_jars
18 | # Fix permissions
19 | - file: >
20 |     path="{{ elasticsearch_home_dir }}/lib" state=directory
21 |     owner={{ elasticsearch_user }} group={{ elasticsearch_group }}
22 |     recurse=yes
23 | 


--------------------------------------------------------------------------------
/vars/vagrant.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Elasticsearch Ansible Variables
 3 | 
 4 | elasticsearch_version: 1.7.3
 5 | elasticsearch_apt_java_package: oracle-java8-installer
 6 | elasticsearch_java_home: /usr/lib/jvm/java-8-oracle
 7 | elasticsearch_heap_size: 1g
 8 | elasticsearch_max_open_files: 65535
 9 | elasticsearch_timezone: "Europe/Oslo"
10 | elasticsearch_node_max_local_storage_nodes: 1
11 | elasticsearch_index_mapper_dynamic: "true"
12 | elasticsearch_memory_bootstrap_mlockall: "true"
13 | elasticsearch_install_java: "true"
14 | elasticsearch_plugins:
15 |   - { name: 'elasticsearch/elasticsearch-mapper-attachments/2.7.1', reinstall: false }
16 |   - { name: 'com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/2.0.9', reinstall: false }
17 |   - { name: 'facet-script', url: 'http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-facet-script-1.1.2.zip', reinstall: false }
18 |   - { name: 'http-basic', url: 'https://github.com/Asquera/elasticsearch-http-basic/releases/download/v1.5.1/elasticsearch-http-basic-1.5.1.jar', download_only: true, reinstall: false }
19 | elasticsearch_thread_pools:
20 |   - "threadpool.bulk.type: fixed"
21 |   - "threadpool.bulk.size: 50"
22 |   - "threadpool.bulk.queue_size: 1000"
23 | elasticsearch_service_startonboot: yes
24 | 


--------------------------------------------------------------------------------
/tasks/aws.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Install AWS Plugin (see https://github.com/elasticsearch/elasticsearch-cloud-aws)
 3 | #
 4 | # The following variables need to be defined in your playbook or inventory:
 5 | # - elasticsearch_plugin_aws_version
 6 | #
 7 | # The following variables provide a for now limited configuration for the plugin.
 8 | # More options may be available in the future.
 9 | # (see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html):
10 | # - elasticsearch_plugin_aws_ec2_groups
11 | # - elasticsearch_plugin_aws_ec2_ping_timeout
12 | # - elasticsearch_plugin_aws_access_key
13 | # - elasticsearch_plugin_aws_secret_key
14 | # - elasticsearch_plugin_aws_region
15 | 
16 | - name: aws | Removing AWS Plugin if it exists
17 |   shell: bin/plugin --remove cloud-aws
18 |     chdir={{ elasticsearch_home_dir }}
19 |   when: elasticsearch_plugin_aws_reinstall is defined and elasticsearch_plugin_aws_reinstall == True
20 |   ignore_errors: yes
21 | - name: aws | Installing AWS Plugin
22 |   shell: bin/plugin -install elasticsearch/elasticsearch-cloud-aws/{{ elasticsearch_plugin_aws_version }}
23 |     chdir={{ elasticsearch_home_dir }}
24 |   register: aws_plugins_installed
25 |   changed_when: "'Installed' in aws_plugins_installed.stdout"
26 |   failed_when: "aws_plugins_installed.rc != 0 and aws_plugins_installed.stdout.find('already exists. To update the plugin') == -1"
27 | 


--------------------------------------------------------------------------------
/defaults/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Elasticsearch Ansible Variables
 3 | 
 4 | elasticsearch_user: elasticsearch
 5 | elasticsearch_group: elasticsearch
 6 | elasticsearch_download_url: https://download.elasticsearch.org/elasticsearch/elasticsearch
 7 | elasticsearch_version: 1.7.3
 8 | elasticsearch_apt_repos:
 9 |   - 'ppa:webupd8team/java'
10 | elasticsearch_apt_java_package: oracle-java8-installer
11 | elasticsearch_apt_dependencies:
12 |   - htop
13 |   - ntp
14 |   - unzip
15 | elasticsearch_max_open_files: 65535
16 | elasticsearch_home_dir: /usr/share/elasticsearch
17 | elasticsearch_plugin_dir: /usr/share/elasticsearch/plugins
18 | elasticsearch_log_dir: /var/log/elasticsearch
19 | elasticsearch_data_dir: /var/lib/elasticsearch
20 | elasticsearch_work_dir: /tmp/elasticsearch
21 | elasticsearch_conf_dir: /etc/elasticsearch
22 | elasticsearch_pid_dir: /var/run
23 | elasticsearch_service_startonboot: no
24 | elasticsearch_timezone: "Etc/UTC"  # Default to UTC
25 | 
26 | #elasticsearch_http_cors_enabled: "false"
27 | elasticsearch_service_state: started
28 | 
29 | # Non-Elasticsearch Defaults
30 | elasticsearch_timezone: UTC
31 | apt_cache_valid_time: 300 # seconds between "apt-get update" calls.
32 | elasticsearch_install_java: "true"
33 | #elasticsearch_repo_dir: path to snapshot restore location
34 | #uncomment to enable shield
35 | #elasticsearch_plugin_shield_enabled: "true"
36 | 
37 | elasticsearch_repo_name: backups
38 | elasticsearch_backup_host: http://localhost:9200
39 | 
40 | 


--------------------------------------------------------------------------------
/tasks/spm.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # SPM Tasks (see http://sematext.com/spm/)
 3 | 
 4 | - name: spm | Install Collectd
 5 |   apt:
 6 |     pkg=collectd
 7 |     state=present
 8 | 
 9 |   # TODO: Make idempotent
10 | - name: spm |  Downloading and running package
11 |   shell: wget --no-check-certificate -O installer.sh "https://apps.sematext.com/spm-reports/installerDownload.do?client_token={{ spm_client_token }}" && bash installer.sh
12 | 
13 |   # TODO: Make idempotent
14 | - name:  spm | Preparing monitor configurations
15 |   shell: bash /opt/spm/bin/spm-client-setup-conf.sh {{ spm_client_token }} es standalone
16 | 
17 | - name:  spm | Configuring JXM in Elasticsearch
18 |   lineinfile:
19 |     dest=/etc/default/elasticsearch
20 |     regexp='^(ES_JAVA_OPTS="\$ES_JAVA_OPTS -Dcom.sun.management.jmxremote.*")$'
21 |     insertafter='^(#ES_JAVA_OPTS=.*)$'
22 |     line='ES_JAVA_OPTS="\$ES_JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=3000 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"'
23 |   notify: Restart Elasticsearch
24 | 
25 | - name: spm | Configuring JXM in SPM
26 |   lineinfile:
27 |     dest=/opt/spm/spm-monitor/conf/spm-monitor-config-{{ spm_client_token }}-default.properties
28 |     regexp='^(SPM_MONITOR_JMX_PARAMS="")$'
29 |     line='SPM_MONITOR_JMX_PARAMS="-Dspm.remote.jmx.url=localhost:3000"'
30 | 
31 | - name: spm | Restarting SPM
32 |   service:
33 |     name=spm-monitor
34 |     state=restarted
35 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | $script = <<SCRIPT
 4 | cd /vagrant
 5 | ansible-playbook -i vagrant-inventory.ini vagrant-main.yml -vv
 6 | SCRIPT
 7 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 8 | VAGRANTFILE_API_VERSION = "2"
 9 | 
10 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
11 |   # Every Vagrant virtual environment requires a box to build off of.
12 |   config.vm.box = "ubuntu/trusty64"
13 |   config.ssh.shell = "bash -c 'BASH_ENV=/etc/profile exec bash'"
14 |   config.vm.provider "virtualbox" do |v|
15 |     v.memory = 2048
16 |     v.cpus = 2
17 |   end
18 | 
19 |   # Create a private network, which allows host-only access to the machine
20 |   # using a specific IP.
21 |   # config.vm.network :private_network, ip: "192.168.33.10"
22 |   config.vm.network :private_network, ip: "192.168.111.10"
23 | 
24 |   # Share an additional folder to the guest VM. The first argument is
25 |   # the path on the host to the actual folder. The second argument is
26 |   # the path on the guest to mount the folder. And the optional third
27 |   # argument is a set of non-required options.
28 |   # config.vm.synced_folder "../data", "/vagrant_data"
29 |   config.vm.synced_folder '.', '/vagrant', mount_options: ["dmode=777,fmode=666"]
30 | 
31 |   config.vm.provision :shell,
32 |     :keep_color => true,
33 |     :path => "setup.sh"
34 |   
35 | 
36 |   config.vm.provision :shell,
37 |    :keep_color => true,
38 |    :inline =>  $script
39 |   
40 | end


--------------------------------------------------------------------------------
/tasks/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Elasticsearch Ansible Tasks
 3 | 
 4 | # Install Java
 5 | - include: java.yml
 6 |   when: elasticsearch_install_java
 7 | 
 8 | # Configure timezome
 9 | - include: timezone.yml
10 | 
11 | # Install and configure elasticsearch
12 | - include: elastic-install.yml
13 | 
14 | # Install AWS Plugin
15 | - include: aws.yml
16 |   when: (elasticsearch_plugin_aws_version is defined)
17 | 
18 | # Install Other Generic Plugins
19 | - include: plugins.yml
20 |   when: (elasticsearch_plugins is defined)
21 | 
22 | # Install custom JARs
23 | - include: custom-jars.yml
24 |   when: (elasticsearch_custom_jars is defined)
25 | 
26 | # restore elasticsearch snapshot
27 | - name: restore elasticsearch snapshot
28 |   include: es_restore.yml
29 |   tags: elasticsearch_restore
30 |   when: elasticsearch_snapshot_restore_name is defined
31 | 
32 | # create  elasticsearch snapshot
33 | - name: restore elasticsearch snapshot
34 |   include: es_snapshot.yml
35 |   tags: elasticsearch_snapshot
36 |   when: elasticsearch_snapshot_name is defined
37 | 
38 | # Setup Shield
39 | - include: shield.yml
40 |   when: (elasticsearch_shield_enabled is defined and elasticsearch_shield_enabled == "true")
41 | 
42 | 
43 | 
44 | - name: make sure config lang dir exists if elasticsearch_lang_config_files
45 |   file: path=/etc/elasticsearch/config/lang/ state=directory
46 |   when: (elasticsearch_lang_config_files is defined)
47 | 
48 | - name: copy in language files
49 |   synchronize: src={{elasticsearch_lang_config_files }} dest=/etc/elasticsearch/config/lang/
50 |   when: (elasticsearch_lang_config_files is defined)
51 |   notify: Restart Elasticsearch
52 | 
53 | 
54 | # Always run post-run tasks
55 | - include: post-run.yml
56 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | language: python
 3 | python: "2.7"
 4 | 
 5 | before_install:
 6 |   # Make sure everything's up to date.
 7 |   - sudo apt-get update -qq
 8 |   - sudo apt-get install -qq python-apt python-pycurl
 9 | 
10 | install:
11 |   # Install Ansible.
12 |   - pip install ansible
13 | 
14 | script:
15 |     - cd tests
16 | 
17 |     - echo -e "\e[31m######AnsibleVersion#######\e[0m"
18 |     - ansible --version
19 | 
20 |     # 1st: check syntax
21 |     - echo "\e[31m######***************************** SYNTAX CHECK (1) *****************************\e[0m"
22 |     - ansible-playbook -i local.ini test1.yml --syntax-check
23 | 
24 |     # 2nd:  Make sure we run the entire playbook
25 |     - echo "***************************** RUN PLAY (2) *****************************"
26 |     - ansible-playbook -i local.ini test1.yml --sudo -vvvv --diff
27 |     - sudo netstat -antlp | grep LISTEN
28 |     - sudo ps aux | grep java
29 |     - curl 127.0.0.1:9200
30 | 
31 |     # 3rd:  Make sure our playbook is idempotent
32 |     - echo "***************************** Idempotence test (3) *****************************"
33 |     - >
34 |       ansible-playbook -i local.ini test1.yml --sudo -vvvv --diff | tee ansible_output
35 |       | grep -q 'changed=0.*failed=0'
36 |       && (echo 'Idempotence test: pass' && exit 0)
37 |       || (echo 'Idempotence test: fail' && exit 1)
38 | 
39 |     # 4th:  Application test
40 |     - ./elastic_test.sh
41 | 
42 | after_failure:
43 |     - echo -e "\e[31m######IdepotanceLog#######\e[0m"
44 |     - sudo cat ansible_output
45 |     - echo -e "\e[31m######netstat#######\e[0m"
46 |     - sudo netstat -atnlp
47 |     - echo -e "\e[31m######ps#######\e[0m"
48 |     - sudo ps aux | grep java
49 |     - echo -e "\e[31m######CurlElasticsearch#######\e[0m"
50 |     - curl 127.0.0.1:9200
51 |     - echo -e "\e[31m######AnsibleFacts#######\e[0m"
52 |     - ansible -i 127.0.0.1, -m setup all -c local
53 | 
54 | after_success:
55 |  - echo -e "\e[0;32m######Cool Success#######\e[0m"
56 | 


--------------------------------------------------------------------------------
/templates/elasticsearch.default.j2:
--------------------------------------------------------------------------------
 1 | # Run ElasticSearch as this user ID and group ID
 2 | #ES_USER=elasticsearch
 3 | {% if elasticsearch_user is defined %}
 4 | ES_USER={{ elasticsearch_user }}
 5 | {% endif %}
 6 | #ES_GROUP=elasticsearch
 7 | {% if elasticsearch_group is defined %}
 8 | ES_GROUP={{ elasticsearch_group }}
 9 | {% endif %}
10 | 
11 | # Heap Size (defaults to 256m min, 1g max)
12 | #ES_HEAP_SIZE=2g
13 | {% if elasticsearch_heap_size is defined %}
14 | ES_HEAP_SIZE={{ elasticsearch_heap_size }}
15 | {% endif %}
16 | 
17 | # Heap new generation
18 | #ES_HEAP_NEWSIZE=
19 | 
20 | # max direct memory
21 | #ES_DIRECT_SIZE=
22 | 
23 | # Maximum number of open files, defaults to 65535.
24 | #MAX_OPEN_FILES=65535
25 | {% if elasticsearch_max_open_files is defined %}
26 | MAX_OPEN_FILES={{ elasticsearch_max_open_files }}
27 | {% endif %}
28 | 
29 | # Maximum locked memory size. Set to "unlimited" if you use the
30 | # bootstrap.mlockall option in elasticsearch.yml. You must also set
31 | # ES_HEAP_SIZE.
32 | #MAX_LOCKED_MEMORY=unlimited
33 | {% if elasticsearch_max_locked_memory is defined %}
34 | MAX_LOCKED_MEMORY={{ elasticsearch_max_locked_memory }}
35 | {% endif %}
36 | 
37 | # ElasticSearch log directory
38 | #LOG_DIR=/var/log/elasticsearch
39 | 
40 | # ElasticSearch data directory
41 | #DATA_DIR=/var/lib/elasticsearch
42 | 
43 | # ElasticSearch work directory
44 | #WORK_DIR=/tmp/elasticsearch
45 | 
46 | # ElasticSearch configuration directory
47 | #CONF_DIR=/etc/elasticsearch
48 | 
49 | # ElasticSearch configuration file (elasticsearch.yml)
50 | #CONF_FILE=/etc/elasticsearch/elasticsearch.yml
51 | 
52 | # Additional Java OPTS
53 | {% if elasticsearch_java_opts is defined %}
54 | ES_JAVA_OPTS="{{ elasticsearch_java_opts }}"
55 | {% else %}
56 | #ES_JAVA_OPTS=
57 | {% endif %}
58 | 
59 | # Environment Vars
60 | {% if elasticsearch_env_use_gc_logging is defined %}
61 | export ES_USE_GC_LOGGING=true
62 | {% endif %}
63 | {% if elasticsearch_java_home is defined %}
64 | JAVA_HOME={{ elasticsearch_java_home }}
65 | {% endif %}
66 | 
67 | PID_DIR={{ elasticsearch_pid_dir }}
68 | 


--------------------------------------------------------------------------------
/templates/elasticsearch.in.sh.j2:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | ES_CLASSPATH=$ES_CLASSPATH:$ES_HOME/lib/elasticsearch-{{ elasticsearch_version }}.jar:$ES_HOME/lib/*:$ES_HOME/lib/sigar/*
 4 | 
 5 | if [ "x$ES_MIN_MEM" = "x" ]; then
 6 |     ES_MIN_MEM=256m
 7 | fi
 8 | if [ "x$ES_MAX_MEM" = "x" ]; then
 9 |     ES_MAX_MEM=1g
10 | fi
11 | if [ "x$ES_HEAP_SIZE" != "x" ]; then
12 |     ES_MIN_MEM=$ES_HEAP_SIZE
13 |     ES_MAX_MEM=$ES_HEAP_SIZE
14 | fi
15 | 
16 | # min and max heap sizes should be set to the same value to avoid
17 | # stop-the-world GC pauses during resize, and so that we can lock the
18 | # heap in memory on startup to prevent any of it from being swapped
19 | # out.
20 | JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"
21 | JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"
22 | 
23 | # new generation
24 | if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then
25 |     JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"
26 | fi
27 | 
28 | # max direct memory
29 | if [ "x$ES_DIRECT_SIZE" != "x" ]; then
30 |     JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"
31 | fi
32 | 
33 | # reduce the per-thread stack size
34 | JAVA_OPTS="$JAVA_OPTS -Xss256k"
35 | 
36 | # set to headless, just in case
37 | JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"
38 | 
39 | # Force the JVM to use IPv4 stack
40 | if [ "x$ES_USE_IPV4" != "x" ]; then
41 |   JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"
42 | fi
43 | 
44 | JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC"
45 | JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC"
46 | 
47 | JAVA_OPTS="$JAVA_OPTS -XX:CMSInitiatingOccupancyFraction=75"
48 | JAVA_OPTS="$JAVA_OPTS -XX:+UseCMSInitiatingOccupancyOnly"
49 | 
50 | # When running under Java 7
51 | # JAVA_OPTS="$JAVA_OPTS -XX:+UseCondCardMark"
52 | 
53 | # GC logging options
54 | if [ "x$ES_USE_GC_LOGGING" != "x" ]; then
55 |   JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"
56 |   JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"
57 |   JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"
58 |   JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"
59 |   JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"
60 | {% if elasticsearch_log_dir is defined %}
61 |   JAVA_OPTS="$JAVA_OPTS -Xloggc:{{ elasticsearch_log_dir }}/gc.log"
62 | {% else %}
63 |   JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/elasticsearch/gc.log"
64 | {% endif %}
65 | {% if elasticsearch_java_opts is defined %}
66 |   JAVA_OPTS="$JAVA_OPTS {{ elasticsearch_java_opts }}"
67 | {% endif %}
68 | fi
69 | 
70 | # Causes the JVM to dump its heap on OutOfMemory.
71 | JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"
72 | # The path to the heap dump location, note directory must exists and have enough
73 | # space for a full heap dump.
74 | #JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof"
75 | 


--------------------------------------------------------------------------------
/tasks/plugins.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Install Elasticsearch Plugins
 3 | #
 4 | # You will need to define an array called 'elasticsearch_plugins' in your playbook or inventory, such that:
 5 | #
 6 | # elasticsearch_plugins:
 7 | #  - { name: '<plugin name>', url: '<[optional] plugin url>' }
 8 | #  - ...
 9 | # where if you were to install the plugin via bin/plugin, you would type:
10 | #
11 | # bin/plugin -install <plugin name>
12 | #
13 | # or
14 | #
15 | # bin/plugin -install <plugin name> -url <plugin url>
16 | 
17 | # Example for https://github.com/elasticsearch/elasticsearch-mapper-attachments (bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/1.9.0):
18 | # elasticsearch_plugins:
19 | #  - { name: 'elasticsearch/elasticsearch-mapper-attachments/1.9.0' }
20 | #
21 | # Example for https://github.com/richardwilly98/elasticsearch-river-mongodb (bin/plugin -i com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/1.7.1):
22 | # elasticsearch_plugins:
23 | #  - { name: 'com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/1.7.1' }
24 | #
25 | # Example for https://github.com/imotov/elasticsearch-facet-script (bin/plugin -install facet-script -url http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-facet-script-1.1.2.zip):
26 | # elasticsearch_plugins:
27 | #  - { name: 'facet-script', url: 'http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-facet-script-1.1.2.zip' }
28 | 
29 | # Loop though elasticsearch_plugins and install them
30 | 
31 | - name: plugins | Removing Old Plugins
32 |   shell: bin/plugin remove {{ item.name }}
33 |     chdir={{ elasticsearch_home_dir }}
34 |   when: item.name is defined and item.reinstall is defined and item.reinstall == True
35 |   with_items: elasticsearch_plugins
36 |   ignore_errors: yes
37 | - name: plugins | Installing Plugins
38 |   shell: bin/plugin install {{ item.name }} {{ '-url ' + item.url if item.url is defined else '' }}
39 |     chdir={{ elasticsearch_home_dir }}
40 |   when: item.download_only is not defined
41 |   with_items: elasticsearch_plugins
42 |   register: plugins_installed
43 |   changed_when: "'Installed' in plugins_installed.stdout"
44 |   failed_when: "plugins_installed.rc != 0 and plugins_installed.stdout.find('already exists. To update the plugin') == -1"
45 | - name: plugins | Ensure Paths for Download only Plugins
46 |   file:
47 |     path="{{ elasticsearch_plugin_dir }}/{{ item.name }}"
48 |     state=directory
49 |   when: item.download_only is defined and item.name is defined
50 |   with_items: elasticsearch_plugins
51 | - name: plugins | Download only Plugins
52 |   get_url:
53 |     url={{ item.url }}
54 |     dest={{ elasticsearch_plugin_dir }}/{{ item.name }}
55 |   when: item.download_only is defined and item.name is defined
56 |   with_items: elasticsearch_plugins
57 |   ignore_errors: yes


--------------------------------------------------------------------------------
/library/es_snapshot_restore.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | __author__ = 'cvig'
 3 | 
 4 | import logging
 5 | try:
 6 |     from elasticsearch import Elasticsearch
 7 |     HAS_ELASTIC = True
 8 | except ImportError:
 9 |     HAS_ELASTIC = False
10 | 
11 | class SnapRestore():
12 |     def __init__(self, module):
13 |         self.backup_name = module.params['repository_name']
14 |         self.es = Elasticsearch(module.params['elasticsearch_host'])
15 |         self.snapshot_indices = module.params['snapshot_indices']
16 |         self.snapshot_name = module.params['snapshot_name']
17 |         self.repository_path = module.params['repository_path']
18 |         self.module = module
19 |         esup = self.es.ping()
20 |         if esup is False:
21 |             msg = "could not connect to  host  %s" % module.params['elasticsearch_host']
22 |             module.exit_json(failed=True, msg=msg)
23 |         self.create_repo()
24 | 
25 |     def create_snapshot(self):
26 |         try:
27 |             body = {}
28 |             if self.snapshot_indices:
29 |                 body['indices'] = self.snapshot_indices
30 |             self.es.snapshot.create(repository=self.backup_name, snapshot=self.snapshot_name, body=body)
31 |         except elasticsearch.ElasticsearchException as ex:
32 |             msg = "Error creating snapshot  %s" % ex
33 |             self.module.exit_json(failed=True, msg=msg)
34 |         except Exception as e:
35 |             msg = "Error creating snapshot  %s" % e
36 |             self.module.exit_json(failed=True, msg=msg)
37 | 
38 |     def restore_snapshot(self):
39 |         try:
40 |             body = {}
41 |             if self.snapshot_indices:
42 |                 body['indices'] = self.snapshot_indices
43 |             self.es.snapshot.restore(repository=self.backup_name, snapshot=self.snapshot_name, body=body)
44 |         except elasticsearch.ElasticsearchException as ex:
45 |             msg = "Error restoring snapshot  %s" % ex
46 |             self.module.exit_json(failed=True, msg=msg)
47 | 
48 |     def create_repo(self):
49 |         try:
50 |             result = self.es.snapshot.get_repository(repository=self.backup_name, ignore=404)
51 |             if not('status' in result):
52 |                 if result[self.backup_name]['settings']['location'] == self.repository_path:
53 |                     msg = "repo already registered at location  %s" % self.repository_path
54 |                     return
55 | 
56 |             body = {'type': 'fs', 'settings': {'location': self.repository_path, 'compress': True}}
57 |             result = self.es.snapshot.create_repository(repository=self.backup_name, body=body, verify=True, ignore=[404, 400])
58 |             msg = "Snapshot repo registered at location  %s output: %s" % (self.repository_path, result)
59 |         except elasticsearch.ElasticsearchException as ex:
60 |             msg = "Error creating snapshot  %s error; %s" % (module.params['elasticsearch_host'], ex)
61 |             self.module.exit_json(failed=True, msg=msg)
62 | 
63 |         #
64 |         # except Exception as err:
65 |         #    print err
66 |         #    module.exit_json(failed=True, msg="There was some kind of error %s" % err)
67 | 
68 | 
69 | def main():
70 |     module = AnsibleModule(
71 |         argument_spec=dict(
72 |             state=dict(default='present', choices=['present', 'absent']),
73 |             repository_name=dict(required=True, type='str'),
74 |             elasticsearch_host=dict(required=True, type='str'),
75 |             snapshot_indices=dict(required=False, type='str'),
76 |             snapshot_name=dict(required=True, type='str'),
77 |             repository_path=dict(required=True, type='str'),
78 |             mode=dict(required=True, choices=['snapshot', 'restore'], type='str')
79 |         )
80 |     )
81 |     if not HAS_ELASTIC:
82 |         module.fail_json(msg='The elasticsearch python module is required')
83 | 
84 |     mode = module.params['mode']
85 |     snaprestore = SnapRestore(module)
86 |     if mode == 'snapshot':
87 |         snaprestore.create_repo()
88 |         snaprestore.create_snapshot()
89 |         module.exit_json(changed=True, result="snapshot '%s' created" % module.params['snapshot_name'])
90 |     if mode == 'restore':
91 |         snaprestore.restore_snapshot()
92 |         module.exit_json(changed=True, result="snapshot '%s' restored" % module.params['snapshot_name'])
93 | 
94 | # import module snippets
95 | from ansible.module_utils.basic import *
96 | logging.basicConfig(level=logging.DEBUG)
97 | main()
98 | 


--------------------------------------------------------------------------------
/tasks/elastic-install.yml:
--------------------------------------------------------------------------------
  1 | ---
  2 | 
  3 | - name: elastic-install | Install python-software-properties
  4 |   apt:
  5 |     pkg=python-software-properties
  6 |     state=present
  7 |     update_cache=yes
  8 |     cache_valid_time={{apt_cache_valid_time}}
  9 | 
 10 | - name: elastic-install | Install dependencies
 11 |   apt:
 12 |     pkg={{ item }}
 13 |     state=present
 14 |   with_items: elasticsearch_apt_dependencies
 15 | 
 16 | - name: elastic-install | Configuring elastic group
 17 |   group:
 18 |     name={{ elasticsearch_group }}
 19 | 
 20 | - name: elastic-install | Configuring elastic user
 21 |   user:
 22 |     name={{ elasticsearch_user }}
 23 |     group={{ elasticsearch_group }}
 24 |     createhome=no
 25 | 
 26 | - name: elastic-install | Ensure temp elasticsearch directories exists
 27 |   file:
 28 |     path="{{ elasticsearch_work_dir }}"
 29 |     state=directory
 30 |     owner={{ elasticsearch_user }}
 31 |     group={{ elasticsearch_group }}
 32 |     recurse=yes
 33 | 
 34 | - name: elastic-install | Check if we have elastic with same version installed
 35 |   stat:
 36 |     path="/usr/share/elasticsearch/lib/elasticsearch-{{ elasticsearch_version }}.jar"
 37 |   register: installed_version
 38 | 
 39 | - name: elastic-install | Try to stop elasticsearch if running
 40 |   service:
 41 |     name=elasticsearch
 42 |     state=stopped
 43 |   ignore_errors: yes
 44 |   when: not installed_version.stat.exists
 45 | 
 46 | - name: elastic-install | Download Elasticsearch deb
 47 |   get_url:
 48 |     url={{ elasticsearch_download_url }}/elasticsearch-{{ elasticsearch_version }}.deb
 49 |     dest=/tmp/elasticsearch-{{ elasticsearch_version }}.deb
 50 |     mode=0440
 51 |   when: not installed_version.stat.exists
 52 | 
 53 | #shell: dpkg --remove elasticsearch
 54 | - name: elastic-install | Uninstalling previous version if applicable
 55 |   apt:
 56 |      name="elasticsearch"
 57 |      state="absent"
 58 |   when: not installed_version.stat.exists
 59 | 
 60 | - name: elastic-install | Remove elasticsearch directory
 61 |   file:
 62 |     path="{{ elasticsearch_home_dir }}"
 63 |     state=absent
 64 |   when: not installed_version.stat.exists
 65 | 
 66 | - name: elastic-install | Install Elasticsearch deb
 67 |   shell: dpkg -i -E --force-confnew /tmp/elasticsearch-{{ elasticsearch_version }}.deb
 68 |   when: not installed_version.stat.exists
 69 |   notify: Restart Elasticsearch
 70 | 
 71 | - name: elastic-install | Ensure elastic directories exists
 72 |   file:
 73 |     path="{{ item }}"
 74 |     state=directory
 75 |     owner={{ elasticsearch_user }}
 76 |     group={{ elasticsearch_group }}
 77 |     recurse=yes
 78 |   with_items:
 79 |      - "{{ elasticsearch_work_dir }}"
 80 |      - "{{ elasticsearch_home_dir }}"
 81 |      - "{{ elasticsearch_log_dir }}"
 82 |      - "{{ elasticsearch_data_dir }}"
 83 |      - "{{ elasticsearch_work_dir }}"
 84 |      - "{{ elasticsearch_conf_dir }}"
 85 | 
 86 | - name: Configure limits max_open_files
 87 |   lineinfile:
 88 |     dest=/etc/security/limits.conf
 89 |     regexp='^{{ elasticsearch_user }}     -    nofile    {{ elasticsearch_max_open_files }}'
 90 |     insertafter=EOF
 91 |     line='{{ elasticsearch_user }}     -    nofile    {{ elasticsearch_max_open_files }}'
 92 |   when: elasticsearch_max_open_files is defined
 93 |   notify: Restart Elasticsearch
 94 | 
 95 | - name: elastic-install | Configure limits max_locked_memory
 96 |   lineinfile:
 97 |     dest=/etc/security/limits.conf
 98 |     regexp='^{{ elasticsearch_user }}     -    memlock   {{ elasticsearch_max_locked_memory }}'
 99 |     insertafter=EOF
100 |     line='{{ elasticsearch_user }}     -    memlock   {{ elasticsearch_max_locked_memory }}'
101 |   when: elasticsearch_max_locked_memory is defined
102 |   notify: Restart Elasticsearch
103 | 
104 | - name: elastic-install | Configure su pam_limits.so
105 |   lineinfile:
106 |     dest=/etc/pam.d/su
107 |     regexp='^session    required   pam_limits.so'
108 |     insertafter=EOF
109 |     line='session    required   pam_limits.so'
110 |   notify: Restart Elasticsearch
111 | 
112 | - name: elastic-install | Configure common-session pam_limits.so
113 |   lineinfile:
114 |     dest=/etc/pam.d/common-session
115 |     regexp='^session    required   pam_limits.so'
116 |     insertafter=EOF
117 |     line='session    required   pam_limits.so'
118 |   notify: Restart Elasticsearch
119 | 
120 | - name: elastic-install | Configure common-session-noninteractive pam_limits.so
121 |   lineinfile:
122 |     dest=/etc/pam.d/common-session-noninteractive
123 |     regexp='^session    required   pam_limits.so'
124 |     insertafter=EOF
125 |     line='session    required   pam_limits.so'
126 |   notify: Restart Elasticsearch
127 | 
128 | - name: elastic-install | Configure sudo pam_limits.so
129 |   lineinfile:
130 |     dest=/etc/pam.d/sudo
131 |     regexp='^session    required   pam_limits.so'
132 |     insertafter=EOF
133 |     line='session    required   pam_limits.so'
134 |   notify: Restart Elasticsearch
135 | 
136 | - name: elastic-install | Configure initd java opts in /etc/init.d/elasticsearch
137 |   lineinfile:
138 |     dest=/etc/init.d/elasticsearch
139 |     regexp='^(DAEMON_OPTS=".*-Des.max-open-files=true")$'
140 |     insertafter='^(DAEMON_OPTS=".*CONF_DIR")$'
141 |     line='DAEMON_OPTS="-Des.max-open-files=true $DAEMON_OPTS"'
142 |   notify: Restart Elasticsearch
143 | 
144 | - name: elastic-install | Configuring Elasticsearch elasticsearch.yml Node
145 |   template:
146 |     src=elasticsearch.yml.j2
147 |     dest={{ elasticsearch_conf_dir }}/elasticsearch.yml
148 |     owner={{ elasticsearch_user }}
149 |     group={{ elasticsearch_group }}
150 |     mode=0644
151 |   when: elasticsearch_conf_dir is defined
152 |   notify: Restart Elasticsearch
153 | 
154 | - name : elastic-install | Configure /etc/default/elasticsearch
155 |   template:
156 |     src=elasticsearch.default.j2
157 |     dest=/etc/default/elasticsearch
158 |     owner={{ elasticsearch_user }}
159 |     group={{ elasticsearch_group }}
160 |     mode=0644
161 |   notify: Restart Elasticsearch


--------------------------------------------------------------------------------
/library/es_reindex.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | __author__ = 'cvig'
  3 | # run this as an ad-hoc ansible task:
  4 | # ansible master -m es_reindex -i /etc/ansible/localhost.ini -a "to_index=wiki2 from_index=wiki"
  5 | 
  6 | import logging
  7 | 
  8 | from ansible.module_utils.basic import *
  9 | ###### MOVE THIS TO A SEPARATE MODULE PLEASE #####
 10 | import logging
 11 | import elasticsearch
 12 | from elasticsearch import Elasticsearch
 13 | from elasticsearch.helpers import scan, bulk
 14 | string_types = str,bytes
 15 | 
 16 | 
 17 | def reindex_h(client, source_index, target_index, query=None, target_client=None,
 18 |         chunk_size=500, scroll='5m', userfields=[], scan_kwargs={}, bulk_kwargs={}):
 19 | 
 20 |     """
 21 |     Reindex all documents from one index that satisfy a given query
 22 |     to another, potentially (if `target_client` is specified) on a different cluster.
 23 |     If you don't specify the query you will reindex all the documents.
 24 | 
 25 |     .. note::
 26 | 
 27 |         This helper doesn't transfer mappings, just the data.
 28 | 
 29 |     :arg client: instance of :class:`~elasticsearch.Elasticsearch` to use (for
 30 |         read if `target_client` is specified as well)
 31 |     :arg source_index: index (or list of indices) to read documents from
 32 |     :arg target_index: name of the index in the target cluster to populate
 33 |     :arg query: body for the :meth:`~elasticsearch.Elasticsearch.search` api
 34 |     :arg target_client: optional, is specified will be used for writing (thus
 35 |         enabling reindex between clusters)
 36 |     :arg chunk_size: number of docs in one chunk sent to es (default: 500)
 37 |     :arg scroll: Specify how long a consistent view of the index should be
 38 |         maintained for scrolled search
 39 |     :arg scan_kwargs: additional kwargs to be passed to
 40 |         :func:`~elasticsearch.helpers.scan`
 41 |     :arg bulk_kwargs: additional kwargs to be passed to
 42 |         :func:`~elasticsearch.helpers.bulk`
 43 |     """
 44 |     target_client = client if target_client is None else target_client
 45 | 
 46 |     docs = scan(client,
 47 |         query=query,
 48 |         index=source_index,
 49 |         scroll=scroll,
 50 |         fields= ('_source', '_parent', '_routing', '_timestamp'),
 51 |         **scan_kwargs
 52 |     )
 53 |     def _change_doc_index(hits, index):
 54 | 
 55 |         for h in hits:
 56 |             h['_index'] = index
 57 |             if 'fields' in h:
 58 |                 h.update(h.pop('fields'))
 59 |             yield h
 60 | 
 61 |     def expand_my_action(data):
 62 |         """
 63 |         From one document or action definition passed in by the user extract the
 64 |         action/data lines needed for elasticsearch's
 65 |         :meth:`~elasticsearch.Elasticsearch.bulk` api.
 66 |         """
 67 |         # when given a string, assume user wants to index raw json
 68 |         if isinstance(data, string_types):
 69 |             return '{"index":{}}', data
 70 | 
 71 |         # make sure we don't alter the action
 72 |         data = data.copy()
 73 |         op_type = data.pop('_op_type', 'index')
 74 |         action = {op_type: {}}
 75 |         for key in ('_index', '_parent', '_percolate', '_routing', '_timestamp',
 76 |                 '_ttl', '_type', '_version', '_version_type', '_id', '_retry_on_conflict'):
 77 |             if key in data:
 78 |                 action[op_type][key] = data.pop(key)
 79 | 
 80 |         # no data payload for delete
 81 |         if op_type == 'delete':
 82 |             return action, None
 83 |         if userfields:
 84 |             returdata = {}
 85 |             for field in userfields:
 86 |                 if field in data['_source']:
 87 |                     returdata[field] = data['_source'][field]
 88 |                 else:
 89 |                     logging.info('field not found in _source "%s"' % field)
 90 | 
 91 | 
 92 |             return action, returdata
 93 |         return action, data.get('_source', data)
 94 | 
 95 | 
 96 |     kwargs = {
 97 |         'stats_only': True,
 98 |         'expand_action': expand_my_action
 99 |     }
100 |     kwargs.update(bulk_kwargs)
101 |     return bulk(target_client, _change_doc_index(docs, target_index),
102 |                 chunk_size=chunk_size, stats_only=True, expand_action_callback=expand_my_action)
103 | 
104 | 
105 | class ReIndex():
106 |     def __init__(self, module):
107 |         if 'elasticsearch_host' in module.params:
108 |             self.es = Elasticsearch(module.params['elasticsearch_host'], timeout=100)
109 |         else:
110 |             self.es = Elasticsearch(timeout=100)
111 |         self.from_index = module.params['from_index']
112 |         self.to_index = module.params['to_index']
113 |         self.query = None
114 |         if 'query' in module.params:
115 |             self.query = module.params['query']
116 |         self.fields = None
117 |         if 'fields' in module.params:
118 |             self.fields = module.params['fields']
119 |         self.module = module
120 |         esup = self.es.ping()
121 |         logging.info('in reindex')
122 |         if not esup:
123 |             msg = "could not connect to  host  %s" % module.params['elasticsearch_host']
124 |             module.exit_json(failed=True, msg=msg)
125 | 
126 |     def reindex(self):
127 |         try:
128 |             reindex_h(self.es, self.from_index, self.to_index, self.query, userfields=self.fields)
129 |         except elasticsearch.ElasticsearchException as ex:
130 |             msg = "Error reindexing   %s" % ex
131 |             self.module.exit_json(failed=True, msg=msg)
132 |         except Exception as e:
133 |             msg = "Error reindexing  %s" % e
134 |             self.module.exit_json(failed=True, msg=msg)
135 | 
136 | 
137 | #class MyModule():
138 | #    def __init__(self, params):
139 | #        self.params = params
140 | #    def exit_json(self, **kwargs):
141 | #        logging.info('feiled')
142 | #def main():
143 | #    module=MyModule({'from_index':'wiki', 'to_index': 'wiki3', 'fields': ['title', 'article', 'timestamp']})
144 | #    reindex=ReIndex(module)
145 | #    reindex.reindex()
146 | 
147 | #logging.basicConfig(filename='reindex.log', level=logging.INFO)
148 | #main()
149 | ### END OF MOVE TO SEPARATE MODULE #####
150 | 
151 | 
152 | def main():
153 | 
154 |     module = AnsibleModule(
155 |         argument_spec=dict(
156 |             from_index=dict(required=True, type='str'),
157 |             to_index=dict(required=True, type='str'),
158 |             fields=dict(required=False, type='list'),
159 |             elasticsearch_host=dict(required=False, type='str', default='localhost:9200'),
160 |             query=dict(required=False, type='str', default=None)
161 |         )
162 |     )
163 |     logging.debug('in main')
164 |     reindex = ReIndex(module)
165 |     reindex.reindex()
166 |     module.exit_json(changed=True, result="Reindex from '%s' to '%s' " % (module.params['from_index'], module.params['to_index']))
167 | 
168 | if __name__ == '__main__':
169 |     logging.basicConfig(filename='example.log', level=logging.INFO)
170 |     main()
171 | 
172 | 
173 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Ansible Playbook for Elasticsearch
  2 | This is an [Ansible](http://www.ansibleworks.com/) playbook for [Elasticsearch](http://www.elasticsearch.org/). You can use it by itself or as part of a larger playbook customized for your local environment.
  3 | 
  4 | ## Features
  5 | - Support for installing plugins
  6 | - Support for installing and configuring EC2 plugin
  7 | - Support for installing custom JARs in the Elasticsearch classpath (e.g. custom Lucene Similarity JAR)
  8 | - Support for installing the [Sematext SPM](http://www.sematext.com/spm/) monitor
  9 | - Support for installing the [Marvel](http://www.elasticsearch.org/guide/en/marvel/current/) plugin
 10 | - Support for installing the [Shield](http://www.elasticsearch.org/guide/en/marvel/current/) plugin
 11 | - Support for Snapshot/restore of indexes
 12 | - Support for Reindexing
 13 | 
 14 | ## Testing locally with Vagrant
 15 | A sample [Vagrant](http://www.vagrantup.com/) configuration is provided to help with local testing. After installing Vagrant, run `vagrant up` at the root of the project to get an VM instance bootstrapped and configured with a running instance of Elasticsearch. Look at `vars/vagrant.yml` and `defaults/main.yml` for the variables that will be substituted in `templates/elasticsearch.yml.j2`.
 16 | 
 17 | ## Running Standalone Playbook
 18 | ### Copy Example Files
 19 | Make copies of the following files and rename them to suit your environment. E.g.:
 20 | 
 21 | - vagrant-main.yml => my-playbook-main.yml
 22 | - vagrant-inventory.ini => my-inventory.ini
 23 | - vars/vagrant.yml => vars/my-vars.yml
 24 | 
 25 | Edit the copied files to suit your environment and needs. See examples below.
 26 | 
 27 | ### Edit your my-inventory.ini
 28 | Edit your my-inventory.ini and customize your cluster and node names:
 29 | 
 30 | ```
 31 | #########################
 32 | # Elasticsearch Cluster #
 33 | #########################
 34 | [es_node_1]
 35 | 1.2.3.4.compute-1.amazonaws.com
 36 | [es_node_1:vars]
 37 | elasticsearch_node_name=elasticsearch-1
 38 | 
 39 | [es_node_2]
 40 | 5.6.7.8.compute-1.amazonaws.com
 41 | [es_node_2:vars]
 42 | elasticsearch_node_name=elasticsearch-2
 43 | 
 44 | [es_node_3]
 45 | 9.10.11.12.compute-1.amazonaws.com
 46 | [es_node_3:vars]
 47 | elasticsearch_node_name=elasticsearch-3
 48 | 
 49 | [all_nodes:children]
 50 | es_node_1
 51 | es_node_2
 52 | es_node_3
 53 | 
 54 | [all_nodes:vars]
 55 | elasticsearch_cluster_name=my.elasticsearch.cluster
 56 | elasticsearch_plugin_aws_ec2_groups=MyElasticSearchGroup
 57 | spm_client_token=<your SPM token here>
 58 | ```
 59 | 
 60 | ### Edit your vars/my-vars.yml
 61 | See `vars/sample.yml` and `vars/vagrant.yml` for example variable files. These are the files where you specify Elasticsearch settings and apply certain features such as plugins, custom JARs or monitoring. The best way to enable configurations is to look at `templates/elasticsearch.yml.j2` and see which variables you want to defile in your `vars/my-vars.yml`. See below for configurations regarding EC2, plugins and custom JARs.
 62 | 
 63 | ### Edit your my-playbook-main.yml
 64 | Example `my-playbook-main.yml`:
 65 | 
 66 | ```
 67 | ---
 68 | 
 69 | #########################
 70 | # Elasticsearch install #
 71 | #########################
 72 | 
 73 | - hosts: all_nodes
 74 |   user: $user
 75 |   sudo: yes
 76 | 
 77 |   vars_files:
 78 |     - defaults/main.yml
 79 |     - vars/my-vars.yml
 80 | 
 81 |   tasks:
 82 |     - include: tasks/main.yml
 83 | ```
 84 | 
 85 | ### Launch
 86 | ```
 87 | $  ansible-playbook my-playbook-main.yml -i my-inventory.ini -e user=<your sudo user for the elasticsearch installation>
 88 | ```
 89 | 
 90 | ## Enabling Added Features
 91 | ### Configuring EC2
 92 | The following variables need to be defined in your playbook or inventory:
 93 | 
 94 | - elasticsearch_plugin_aws_version
 95 | 
 96 | See [https://github.com/elasticsearch/elasticsearch-cloud-aws](https://github.com/elasticsearch/elasticsearch-cloud-aws) for the version that most accurately matches your installation.
 97 | 
 98 | The following variables provide a for now limited configuration for the plugin. More options may be available in the future (see [http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html)](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html)):
 99 | 
100 | - elasticsearch_plugin_aws_ec2_groups
101 | - elasticsearch_plugin_aws_ec2_ping_timeout
102 | - elasticsearch_plugin_aws_access_key
103 | - elasticsearch_plugin_aws_secret_key
104 | - elasticsearch_plugin_aws_region
105 | 
106 | ### Installing plugins
107 | You will need to define an array called `elasticsearch_plugins` in your playbook or inventory, such that:
108 | ```
109 | elasticsearch_plugins:
110 |  - { name: '<plugin name>', url: '<[optional] plugin url>' }
111 |  - ...
112 | ```
113 | 
114 | where if you were to install the plugin via bin/plugin, you would type:
115 | `bin/plugin -install <plugin name>` or `bin/plugin -install <plugin name> -url <plugin url>`
116 | 
117 | Example for [https://github.com/elasticsearch/elasticsearch-mapper-attachments](https://github.com/elasticsearch/elasticsearch-mapper-attachments) (`bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/1.9.0`):
118 | 
119 | ```
120 | elasticsearch_plugins:
121 |  - { name: 'elasticsearch/elasticsearch-mapper-attachments/1.9.0' }
122 | ```
123 | 
124 | Example for [https://github.com/richardwilly98/elasticsearch-river-mongodb](https://github.com/richardwilly98/elasticsearch-river-mongodb) (`bin/plugin -i com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/1.7.1`):
125 | 
126 | ```
127 | elasticsearch_plugins:
128 |  - { name: 'com.github.richardwilly98.elasticsearch/elasticsearch-river-mongodb/1.7.1' }
129 | ```
130 | 
131 | Example for [https://github.com/imotov/elasticsearch-facet-script](https://github.com/imotov/elasticsearch-facet-script) (`bin/plugin -install facet-script -url http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-facet-script-1.1.2.zip`):
132 | 
133 | ```
134 | elasticsearch_plugins:
135 |  - { name: 'facet-script', url: 'http://dl.bintray.com/content/imotov/elasticsearch-plugins/elasticsearch-facet-script-1.1.2.zip' }
136 | ```
137 | 
138 | ### Installing Custom JARs
139 | Custom jars are made available to the Elasticsearch classpath by being downloaded into the elasticsearch_home_dir/lib folder. An example of a custom jar can include a custom Lucene Similarity Provider. You will need to define an array called `elasticsearch_custom_jars` in your playbook or inventory, such that:
140 | 
141 | ```
142 | elasticsearch_custom_jars:
143 |  - { uri: '<URL where JAR can be downloaded from: required>', filename: '<alternative name for final JAR if different from file downladed: leave blank to use same filename>', user: '<BASIC auth username: leave blank of not needed>', passwd: '<BASIC auth password: leave blank of not needed>' }
144 |  - ...
145 | ```
146 | 
147 | ### Configuring Thread Pools
148 | Elasticsearch [thread pools](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-threadpool.html) can be configured using the `elasticsearch_thread_pools` list variable:
149 | 
150 | ```
151 | elasticsearch_thread_pools:
152 |   - "threadpool.bulk.type: fixed"
153 |   - "threadpool.bulk.size: 50"
154 |   - "threadpool.bulk.queue_size: 1000"
155 | ```
156 | 
157 | ### Enabling Sematext SPM
158 | Enable the SPM task in your playbook:
159 | 
160 | ```
161 | tasks:
162 |     - include: tasks/spm.yml
163 |     - ...
164 | ```
165 | 
166 | Set the spm_client_token variable in your inventory.ini to your SPM key.
167 | 
168 | ### Configuring Marvel
169 | 
170 | The following variables need to be defined in your playbook or inventory:
171 | 
172 | - elasticsearch_plugin_marvel_version
173 | 
174 | The following variables provide configuration for the plugin. More options may be available in the future (see [http://www.elasticsearch.org/guide/en/marvel/current/#stats-export](http://www.elasticsearch.org/guide/en/marvel/current/#stats-export)):
175 | 
176 | - elasticsearch_plugin_marvel_agent_enabled
177 | - elasticsearch_plugin_marvel_agent_exporter_es_hosts
178 | - elasticsearch_plugin_marvel_agent_indices
179 | - elasticsearch_plugin_marvel_agent_interval
180 | - elasticsearch_plugin_marvel_agent_exporter_es_index_timeformat
181 | 
182 | 
183 | ### Configuring Shield
184 | The following variables need to be defined in your playbook or inventory:
185 | 
186 | - elasticsearch_plugin_shield_enabled
187 | 
188 | The following variables provide configuration for the plugin. More options may be available in the future (see [https://www.elastic.co/guide/en/shield/current/reference.html](https://www.elastic.co/guide/en/shield/current/reference.html)):
189 | - elasticsearch_plugin_shield_ssl_keystore_path
190 | - elasticsearch_plugin_shield_ssl_keystore_password
191 | - elasticsearch_plugin_shield_ssl_keystore_key_password
192 | - elasticsearch_plugin_shield_ssl_hostname_verification
193 | - elasticsearch_plugin_shield_transport_ssl
194 | - elasticsearch_plugin_shield_http_ssl
195 | - elasticsearch_plugin_shield_audit_enabled
196 | - elasticsearch_plugin_shield_realms
197 | - elasticsearch_plugin_shield_realms_esusers
198 |   + order: 0
199 |   + enabled: "false"
200 |    
201 | - elasticsearch_plugin_shield_realms_active_directory
202 |   + order: 1
203 |   + domain_name: example.com
204 |   + unmapped_groups_as_roles: "true"
205 |   + url: ldap://parsecdc005.pareto.no
206 |   + enabled: "true"
207 |    
208 | - elasticsearch_esusers
209 |   + - {username:demouser, password: mypass, role: myrole}
210 |  
211 |  - elasticsearch_shield_files: path to folder with Shield configuration files which will be copied into /etc/elasticsearch/shield
212 | 
213 | ## Snapshot/Restore
214 | 
215 | Use the es_snapshot_restore module as part of your own task like this:
216 | ```
217 |  - name: restore es wiki backups
218 |    es_snapshot_restore: elasticsearch_host=http://127.0.0.1:9200 repository_path="{{elasticsearch_repo_dir}}" repository_name=my_backup snapshot_name="{{snapshot_name}}" snapshot_indices="{{snapshot_indices}}" mode=restore state=present
219 | ```
220 | 
221 | `` {{ elasticsearch_repo_dir }}`` , if present, will be registered in elasticsearch.yml with the config setting path.repo. 
222 | 
223 | 
224 | ## Reindexing
225 | The es_reindex module can be used to transfer data from one index to another. You can use it an an ad-hoc ansible task like this
226 | ```
227 |  ansible indexer1 -m es_reindex -a "to_index=wiki2 from_index=wiki"
228 | ```
229 | ## Disable Java installation
230 | 
231 | If you prefer to skip the built-in installation of the Oracle JRE, use the `elasticsearch_install_java` flag:
232 | 
233 | ```
234 | elasticsearch_install_java: "false"
235 | ```
236 | 
237 | ## Include role in a larger playbook
238 | ### Add this role as a git submodule
239 | Assuming your playbook structure is such as:
240 | ```
241 | - my-master-playbook
242 |   |- vars
243 |   |- roles
244 |   |- my-master-playbook-main.yml
245 |   \- my-master-inventory.ini
246 | ```
247 | 
248 | Checkout this project as a submodule under roles:
249 | 
250 | ```
251 | $  cd roles
252 | $  git submodule add https://github.com/comperiosearch/ansible-elasticsearch.git ./elasticsearch
253 | $  git submodule update --init
254 | $  git commit ./submodule -m "Added submodule as ./subm"
255 | ```
256 | 
257 | ### Include this playbook as a role in your master playbook
258 | Example `my-master-playbook-main.yml`:
259 | 
260 | ```
261 | ---
262 | 
263 | #########################
264 | # Elasticsearch install #
265 | #########################
266 | 
267 | - hosts: all_nodes
268 |   user: ubuntu
269 |   sudo: yes
270 | 
271 |   roles:
272 |     - elasticsearch
273 | 
274 |   vars_files:
275 |     - vars/my-vars.yml
276 | ```
277 | 
278 | # Issues, requests, contributions
279 | This software is provided as is. Having said that, if you see an issue, feel free to log a ticket. We'll do our best to address it. Same if you want to see a certain feature supported in the fututre. No guarantees are made that any requested feature will be implemented. If you'd like to contribute, feel free to clone and submit a pull request.
280 | 
281 | # Dependencies
282 | None
283 | 
284 | # License
285 | MIT
286 | 
287 | # Author Information
288 | 
289 | George Stathis - gstathis [at] traackr.com
290 | Mats Olsen molsen [at] comperiosearch.com
291 | Christoffer Vig cvig [at] comperiosearch.com
292 | 


--------------------------------------------------------------------------------
/templates/elasticsearch.yml.j2:
--------------------------------------------------------------------------------
  1 | ##################### ElasticSearch Configuration Example #####################
  2 | 
  3 | # This file contains an overview of various configuration settings,
  4 | # targeted at operations staff. Application developers should
  5 | # consult the guide at <http://elasticsearch.org/guide>.
  6 | #
  7 | # The installation procedure is covered at
  8 | # <http://elasticsearch.org/guide/reference/setup/installation.html>.
  9 | #
 10 | # ElasticSearch comes with reasonable defaults for most settings,
 11 | # so you can try it out without bothering with configuration.
 12 | #
 13 | # Most of the time, these defaults are just fine for running a production
 14 | # cluster. If you're fine-tuning your cluster, or wondering about the
 15 | # effect of certain configuration option, please _do ask_ on the
 16 | # mailing list or IRC channel [http://elasticsearch.org/community].
 17 | 
 18 | # See <http://elasticsearch.org/guide/reference/setup/configuration.html>
 19 | # for information on supported formats and syntax for the configuration file.
 20 | 
 21 | 
 22 | ################################### Cluster ###################################
 23 | 
 24 | # Cluster name identifies your cluster for auto-discovery. If you're running
 25 | # multiple clusters on the same network, make sure you're using unique names.
 26 | #
 27 | # cluster.name: elasticsearch
 28 | {% if elasticsearch_cluster_name is defined %}
 29 | cluster.name: {{ elasticsearch_cluster_name }}
 30 | {% endif %}
 31 | 
 32 | #################################### Node #####################################
 33 | 
 34 | # Node names are generated dynamically on startup, so you're relieved
 35 | # from configuring them manually. You can tie this node to a specific name:
 36 | #
 37 | # node.name: "Franz Kafka"
 38 | {% if elasticsearch_node_name is defined %}
 39 | node.name: {{ elasticsearch_node_name }}
 40 | {% endif %}
 41 | 
 42 | # Every node can be configured to allow or deny being eligible as the master,
 43 | # and to allow or deny to store the data.
 44 | #
 45 | # Allow this node to be eligible as a master node (enabled by default):
 46 | #
 47 | # node.master: true
 48 | {% if elasticsearch_node_master is defined %}
 49 | node.master: {{ elasticsearch_node_master }}
 50 | {% endif %}
 51 | #
 52 | # Allow this node to store data (enabled by default):
 53 | #
 54 | # node.data: true
 55 | {% if elasticsearch_node_data is defined %}
 56 | node.data: {{ elasticsearch_node_data }}
 57 | {% endif %}
 58 | 
 59 | # You can exploit these settings to design advanced cluster topologies.
 60 | #
 61 | # 1. You want this node to never become a master node, only to hold data.
 62 | #    This will be the "workhorse" of your cluster.
 63 | #
 64 | # node.master: false
 65 | # node.data: true
 66 | #
 67 | # 2. You want this node to only serve as a master: to not store any data and
 68 | #    to have free resources. This will be the "coordinator" of your cluster.
 69 | #
 70 | # node.master: true
 71 | # node.data: false
 72 | #
 73 | # 3. You want this node to be neither master nor data node, but
 74 | #    to act as a "search load balancer" (fetching data from nodes,
 75 | #    aggregating results, etc.)
 76 | #
 77 | # node.master: false
 78 | # node.data: false
 79 | 
 80 | # Use the Cluster Health API [http://localhost:9200/_cluster/health], the
 81 | # Node Info API [http://localhost:9200/_cluster/nodes] or GUI tools
 82 | # such as <http://github.com/lukas-vlcek/bigdesk> and
 83 | # <http://mobz.github.com/elasticsearch-head> to inspect the cluster state.
 84 | 
 85 | # A node can have generic attributes associated with it, which can later be used
 86 | # for customized shard allocation filtering, or allocation awareness. An attribute
 87 | # is a simple key value pair, similar to node.key: value, here is an example:
 88 | #
 89 | # node.rack: rack314
 90 | {% if elasticsearch_node_rack is defined %}
 91 | node.rack: {{ elasticsearch_node_rack }}
 92 | {% endif %}
 93 | 
 94 | # By default, multiple nodes are allowed to start from the same installation location
 95 | # to disable it, set the following:
 96 | # node.max_local_storage_nodes: 1
 97 | {% if elasticsearch_node_max_local_storage_nodes is defined %}
 98 | node.max_local_storage_nodes: {{ elasticsearch_node_max_local_storage_nodes }}
 99 | {% endif %}
100 | 
101 | # Disable network discovery by setting the following (useful for development):
102 | # node.local: true
103 | {% if elasticsearch_node_local is defined %}
104 | node.local: {{ elasticsearch_node_local }}
105 | {% endif %}
106 | 
107 | #################################### Index ####################################
108 | 
109 | # You can set a number of options (such as shard/replica options, mapping
110 | # or analyzer definitions, translog settings, ...) for indices globally,
111 | # in this file.
112 | #
113 | # Note, that it makes more sense to configure index settings specifically for
114 | # a certain index, either when creating it or by using the index templates API.
115 | #
116 | # See <http://elasticsearch.org/guide/reference/index-modules/> and
117 | # <http://elasticsearch.org/guide/reference/api/admin-indices-create-index.html>
118 | # for more information.
119 | 
120 | # Set the number of shards (splits) of an index (5 by default):
121 | #
122 | # index.number_of_shards: 5
123 | {% if elasticsearch_index_number_of_shards is defined %}
124 | index.number_of_shards: {{ elasticsearch_index_number_of_shards }}
125 | {% endif %}
126 | 
127 | # Set the number of replicas (additional copies) of an index (1 by default):
128 | #
129 | # index.number_of_replicas: 1
130 | {% if elasticsearch_index_number_of_replicas is defined %}
131 | index.number_of_replicas: {{ elasticsearch_index_number_of_replicas }}
132 | {% endif %}
133 | 
134 | # Note, that for development on a local machine, with small indices, it usually
135 | # makes sense to "disable" the distributed features:
136 | #
137 | # index.number_of_shards: 1
138 | # index.number_of_replicas: 0
139 | 
140 | # These settings directly affect the performance of index and search operations
141 | # in your cluster. Assuming you have enough machines to hold shards and
142 | # replicas, the rule of thumb is:
143 | #
144 | # 1. Having more *shards* enhances the _indexing_ performance and allows to
145 | #    _distribute_ a big index across machines.
146 | # 2. Having more *replicas* enhances the _search_ performance and improves the
147 | #    cluster _availability_.
148 | #
149 | # The "number_of_shards" is a one-time setting for an index.
150 | #
151 | # The "number_of_replicas" can be increased or decreased anytime,
152 | # by using the Index Update Settings API.
153 | #
154 | # ElasticSearch takes care about load balancing, relocating, gathering the
155 | # results from nodes, etc. Experiment with different settings to fine-tune
156 | # your setup.
157 | 
158 | # Use the Index Status API (<http://localhost:9200/A/_status>) to inspect
159 | # the index status.
160 | 
161 | {% if elasticsearch_index_mapper_dynamic is defined %}
162 | index.mapper_dynamic: {{ elasticsearch_index_mapper_dynamic }}
163 | {% endif %}
164 | {% if elasticsearch_misc_query_bool_max_clause_count is defined %}
165 | index.query.bool.max_clause_count: {{ elasticsearch_misc_query_bool_max_clause_count }}
166 | {% endif %}
167 | {% if elasticsearch_index_refresh_interval is defined %}
168 | index.refresh_interval: {{ elasticsearch_index_refresh_interval }}
169 | {% endif %}
170 | {% if elasticsearch_index_store_throttle_type is defined %}
171 | index.store.throttle.type: {{ elasticsearch_index_store_throttle_type }}
172 | {% endif %}
173 | {% if elasticsearch_index_store_throttle_max_bytes_per_sec is defined %}
174 | indices.store.throttle.max_bytes_per_sec: {{ elasticsearch_index_store_throttle_max_bytes_per_sec }}
175 | {% endif %}
176 | {% if elasticsearch_index_merge_scheduler_max_thread_count is defined %}
177 | index.merge.scheduler.max_thread_count: {{ elasticsearch_index_merge_scheduler_max_thread_count }}
178 | {% endif %}
179 | 
180 | 
181 | #################################### Paths ####################################
182 | 
183 | # Path to directory containing configuration (this file and logging.yml):
184 | #
185 | # path.conf: /path/to/conf
186 | {% if elasticsearch_conf_dir is defined %}
187 | path.conf: {{ elasticsearch_conf_dir }}
188 | {% endif %}
189 | 
190 | # Path to directory where to store index data allocated for this node.
191 | #
192 | # path.data: /path/to/data
193 | {% if elasticsearch_data_dir is defined %}
194 | path.data: {{ elasticsearch_data_dir }}
195 | {% endif %}
196 | 
197 | # Can optionally include more than one location, causing data to be striped across
198 | # the locations (a la RAID 0) on a file level, favouring locations with most free
199 | # space on creation. For example:
200 | #
201 | # path.data: /path/to/data1,/path/to/data2
202 | 
203 | # Path to temporary files:
204 | #
205 | # path.work: /path/to/work
206 | {% if elasticsearch_work_dir is defined %}
207 | path.work: {{ elasticsearch_work_dir }}
208 | {% endif %}
209 | 
210 | # Path to log files:
211 | #
212 | # path.logs: /path/to/logs
213 | {% if elasticsearch_log_dir is defined %}
214 | path.logs: {{ elasticsearch_log_dir }}
215 | {% endif %}
216 | 
217 | # Path to where plugins are installed:
218 | #
219 | # path.plugins: /path/to/plugins
220 | {% if elasticsearch_plugin_dir is defined %}
221 | path.plugins: {{ elasticsearch_plugin_dir }}
222 | {% endif %}
223 | 
224 | 
225 | # Path to snapshot/restore repost
226 | #path.repo: //pppp
227 | {% if elasticsearch_repo_dir is defined %}
228 | path.repo: {{ elasticsearch_repo_dir }}
229 | {% endif %}
230 | 
231 | 
232 | #################################### Plugin ###################################
233 | 
234 | # If a plugin listed here is not installed for current node, the node will not start.
235 | #
236 | # plugin.mandatory: mapper-attachments,lang-groovy
237 | 
238 | 
239 | ################################### Memory ####################################
240 | 
241 | # ElasticSearch performs poorly when JVM starts swapping: you should ensure that
242 | # it _never_ swaps.
243 | #
244 | # Set this property to true to lock the memory:
245 | #
246 | #
247 | {% if elasticsearch_memory_bootstrap_mlockall is defined %}
248 | bootstrap.mlockall: {{ elasticsearch_memory_bootstrap_mlockall }}
249 | {% endif %}
250 | 
251 | # Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set
252 | # to the same value, and that the machine has enough memory to allocate
253 | # for ElasticSearch, leaving enough memory for the operating system itself.
254 | #
255 | # You should also make sure that the ElasticSearch process is allowed to lock
256 | # the memory, eg. by using `ulimit -l unlimited`.
257 | 
258 | {% if elasticsearch_indices_fielddata_cache_size is defined %}
259 | indices.fielddata.cache.size: {{ elasticsearch_indices_fielddata_cache_size }}
260 | {% endif %}
261 | 
262 | {% if elasticsearch_indices_breaker_fielddata_limit is defined %}
263 | indices.breaker.fielddata.limit: {{ elasticsearch_indices_breaker_fielddata_limit }}
264 | {% endif %}
265 | 
266 | {% if elasticsearch_indices_breaker_request_limit is defined %}
267 | indices.breaker.request.limit: {{ elasticsearch_indices_breaker_request_limit }}
268 | {% endif %}
269 | 
270 | {% if elasticsearch_indices_breaker_total_limit is defined %}
271 | indices.breaker.total.limit: {{ elasticsearch_indices_breaker_total_limit }}
272 | {% endif %}
273 | 
274 | ############################## Network And HTTP ###############################
275 | 
276 | # ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens
277 | # on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node
278 | # communication. (the range means that if the port is busy, it will automatically
279 | # try the next port).
280 | 
281 | # Set the bind address specifically (IPv4 or IPv6):
282 | #
283 | # network.bind_host: 192.168.0.1
284 | {% if elasticsearch_network_bind_host is defined %}
285 | network.bind_host: {{ elasticsearch_network_bind_host }}
286 | {% endif %}
287 | 
288 | # Set the address other nodes will use to communicate with this node. If not
289 | # set, it is automatically derived. It must point to an actual IP address.
290 | #
291 | # network.publish_host: 192.168.0.1
292 | {% if elasticsearch_network_publish_host is defined %}
293 | network.publish_host: {{ elasticsearch_network_publish_host }}
294 | {% endif %}
295 | 
296 | # Set both 'bind_host' and 'publish_host':
297 | #
298 | # network.host: 192.168.0.1
299 | {% if elasticsearch_network_host is defined %}
300 | network.host: {{ elasticsearch_network_host }}
301 | {% endif %}
302 | 
303 | # Set a custom port for the node to node communication (9300 by default):
304 | #
305 | # transport.tcp.port: 9300
306 | {% if elasticsearch_network_transport_tcp_port is defined %}
307 | transport.tcp.port: {{ elasticsearch_network_transport_tcp_port }}
308 | {% endif %}
309 | 
310 | # Enable compression for all communication between nodes (disabled by default):
311 | #
312 | # transport.tcp.compress: true
313 | {% if elasticsearch_network_transport_tcp_compress is defined %}
314 | transport.tcp.compress: {{ elasticsearch_network_transport_tcp_compress }}
315 | {% endif %}
316 | 
317 | # Set a custom port to listen for HTTP traffic:
318 | #
319 | # http.port: 9200
320 | {% if elasticsearch_network_http_port is defined %}
321 | http.port: {{ elasticsearch_network_http_port }}
322 | {% endif %}
323 | 
324 | # Set a custom allowed content length:
325 | #
326 | # http.max_content_length: 100mb
327 | {% if elasticsearch_network_http_max_content_lengtht is defined %}
328 | http.max_content_length: {{ elasticsearch_network_http_max_content_lengtht }}
329 | {% endif %}
330 | 
331 | # Disable HTTP completely:
332 | #
333 | # http.enabled: false
334 | {% if elasticsearch_network_http_enabled is defined %}
335 | http.enabled: {{ elasticsearch_network_http_enabled }}
336 | {% endif %}
337 | 
338 | # Basic HTTP
339 | {% if elasticsearch_http_basic_log is defined %}
340 | http.basic.log: {{ elasticsearch_http_basic_log }}
341 | {% endif %}
342 | {% if elasticsearch_http_basic_user is defined %}
343 | http.basic.user: {{ elasticsearch_http_basic_user }}
344 | {% endif %}
345 | {% if elasticsearch_http_basic_password is defined %}
346 | http.basic.password: {{ elasticsearch_http_basic_password }}
347 | {% endif %}
348 | 
349 | 
350 | ################################### Gateway ###################################
351 | 
352 | # The gateway allows for persisting the cluster state between full cluster
353 | # restarts. Every change to the state (such as adding an index) will be stored
354 | # in the gateway, and when the cluster starts up for the first time,
355 | # it will read its state from the gateway.
356 | 
357 | # There are several types of gateway implementations. For more information,
358 | # see <http://elasticsearch.org/guide/reference/modules/gateway>.
359 | 
360 | # The default gateway type is the "local" gateway (recommended):
361 | #
362 | # gateway.type: local
363 | {% if elasticsearch_gateway_type is defined %}
364 | gateway.type: {{ elasticsearch_gateway_type }}
365 | {% endif %}
366 | 
367 | # Settings below control how and when to start the initial recovery process on
368 | # a full cluster restart (to reuse as much local data as possible when using shared
369 | # gateway).
370 | 
371 | # Allow recovery process after N nodes in a cluster are up:
372 | #
373 | # gateway.recover_after_nodes: 1
374 | {% if elasticsearch_gateway_recover_after_nodes is defined %}
375 | gateway.recover_after_nodes: {{ elasticsearch_gateway_recover_after_nodes }}
376 | {% endif %}
377 | 
378 | # Set the timeout to initiate the recovery process, once the N nodes
379 | # from previous setting are up (accepts time value):
380 | #
381 | # gateway.recover_after_time: 5m
382 | {% if elasticsearch_gateway_recover_after_time is defined %}
383 | gateway.recover_after_time: {{ elasticsearch_gateway_recover_after_time }}
384 | {% endif %}
385 | 
386 | # Set how many nodes are expected in this cluster. Once these N nodes
387 | # are up (and recover_after_nodes is met), begin recovery process immediately
388 | # (without waiting for recover_after_time to expire):
389 | #
390 | # gateway.expected_nodes: 2
391 | {% if elasticsearch_gateway_expected_nodes is defined %}
392 | gateway.expected_nodes: {{ elasticsearch_gateway_expected_nodes }}
393 | {% endif %}
394 | 
395 | ############################# Recovery Throttling #############################
396 | 
397 | # These settings allow to control the process of shards allocation between
398 | # nodes during initial recovery, replica allocation, rebalancing,
399 | # or when adding and removing nodes.
400 | 
401 | # Set the number of concurrent recoveries happening on a node:
402 | #
403 | # 1. During the initial recovery
404 | #
405 | # cluster.routing.allocation.node_initial_primaries_recoveries: 4
406 | {% if elasticsearch_recovery_node_initial_primaries_recoveries is defined %}
407 | cluster.routing.allocation.node_initial_primaries_recoveries: {{ elasticsearch_recovery_node_initial_primaries_recoveries }}
408 | {% endif %}
409 | 
410 | #
411 | # 2. During adding/removing nodes, rebalancing, etc
412 | #
413 | # cluster.routing.allocation.node_concurrent_recoveries: 2
414 | {% if elasticsearch_recovery_node_concurrent_recoveries is defined %}
415 | cluster.routing.allocation.node_concurrent_recoveries: {{ elasticsearch_recovery_node_concurrent_recoveries }}
416 | {% endif %}
417 | 
418 | # Set to throttle throughput when recovering (eg. 100mb, by default unlimited):
419 | #
420 | # indices.recovery.max_size_per_sec: 0
421 | {% if elasticsearch_recovery_max_size_per_sec is defined %}
422 | indices.recovery.max_size_per_sec: {{ elasticsearch_recovery_max_size_per_sec }}
423 | {% endif %}
424 | 
425 | # Set to limit the number of open concurrent streams when
426 | # recovering a shard from a peer:
427 | #
428 | # indices.recovery.concurrent_streams: 5
429 | {% if elasticsearch_recovery_concurrent_streams is defined %}
430 | indices.recovery.concurrent_streams: {{ elasticsearch_recovery_concurrent_streams }}
431 | {% endif %}
432 | 
433 | 
434 | ################################## Discovery ##################################
435 | 
436 | {% if elasticsearch_plugin_aws_version is defined %}
437 | 
438 | {% if elasticsearch_plugin_aws_discovery_disable is not defined %}
439 | discovery.type: ec2
440 | {% endif %}
441 | {% if elasticsearch_plugin_aws_ec2_groups is defined %}
442 | discovery.ec2.groups: '{{ elasticsearch_plugin_aws_ec2_groups }}'
443 | {% endif %}
444 | {% if elasticsearch_plugin_aws_ec2_ping_timeout is defined %}
445 | discovery.ec2.ping_timeout: {{ elasticsearch_plugin_aws_ec2_ping_timeout}}
446 | {% endif %}
447 | cloud.node.auto_attributes: true
448 | {% if elasticsearch_plugin_aws_access_key is defined %}
449 | {% if elasticsearch_plugin_aws_secret_key is defined %}
450 | cloud.aws.access_key: '{{ elasticsearch_plugin_aws_access_key }}'
451 | cloud.aws.secret_key: '{{ elasticsearch_plugin_aws_secret_key }}'
452 | {% endif %}
453 | {% endif %}
454 | {% if elasticsearch_plugin_aws_region is defined %}
455 | cloud.aws.region: {{ elasticsearch_plugin_aws_region}}
456 | {% endif %}
457 | 
458 | {% endif %}
459 | 
460 | # Discovery infrastructure ensures nodes can be found within a cluster
461 | # and master node is elected. Multicast discovery is the default.
462 | 
463 | # Set to ensure a node sees N other master eligible nodes to be considered
464 | # operational within the cluster. Set this option to a higher value (2-4)
465 | # for large clusters (>3 nodes):
466 | #
467 | # discovery.zen.minimum_master_nodes: 1
468 | {% if elasticsearch_discovery_zen_minimum_master_nodes is defined %}
469 | discovery.zen.minimum_master_nodes: {{ elasticsearch_discovery_zen_minimum_master_nodes }}
470 | {% endif %}
471 | 
472 | # Set the time to wait for ping responses from other nodes when discovering.
473 | # Set this option to a higher value on a slow or congested network
474 | # to minimize discovery failures:
475 | #
476 | # discovery.zen.ping.timeout: 3s
477 | {% if elasticsearch_discovery_zen_ping_timeout is defined %}
478 | discovery.zen.ping.timeout: {{ elasticsearch_discovery_zen_ping_timeout }}
479 | {% endif %}
480 | 
481 | # See <http://elasticsearch.org/guide/reference/modules/discovery/zen.html>
482 | # for more information.
483 | 
484 | # Unicast discovery allows to explicitly control which nodes will be used
485 | # to discover the cluster. It can be used when multicast is not present,
486 | # or to restrict the cluster communication-wise.
487 | #
488 | # 1. Disable multicast discovery (enabled by default):
489 | #
490 | # discovery.zen.ping.multicast.enabled: false
491 | {% if elasticsearch_discovery_zen_ping_multicast_enabled is defined %}
492 | discovery.zen.ping.multicast.enabled: {{ elasticsearch_discovery_zen_ping_multicast_enabled }}
493 | {% endif %}
494 | 
495 | #
496 | # 2. Configure an initial list of master nodes in the cluster
497 | #    to perform discovery when new nodes (master or data) are started:
498 | #
499 | # discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"]
500 | {% if elasticsearch_discovery_zen_ping_unicast_hosts is defined %}
501 | discovery.zen.ping.unicast.hosts: {{ elasticsearch_discovery_zen_ping_unicast_hosts }}
502 | {% endif %}
503 | 
504 | # EC2 discovery allows to use AWS EC2 API in order to perform discovery.
505 | #
506 | # You have to install the cloud-aws plugin for enabling the EC2 discovery.
507 | #
508 | # See <http://elasticsearch.org/guide/reference/modules/discovery/ec2.html>
509 | # for more information.
510 | #
511 | # See <http://elasticsearch.org/tutorials/2011/08/22/elasticsearch-on-ec2.html>
512 | # for a step-by-step tutorial.
513 | 
514 | # Fault Discovery
515 | # See <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#fault-detection>
516 | {% if elasticsearch_discovery_zen_fd_ping_interval is defined %}
517 | discovery.zen.fd.ping_interval: {{ elasticsearch_discovery_zen_fd_ping_interval }}
518 | {% endif %}
519 | {% if elasticsearch_discovery_zen_fd_ping_timeout is defined %}
520 | discovery.zen.fd.ping_timeout: {{ elasticsearch_discovery_zen_fd_ping_timeout }}
521 | {% endif %}
522 | {% if elasticsearch_discovery_zen_fd_ping_retries is defined %}
523 | discovery.zen.fd.ping_retries: {{ elasticsearch_discovery_zen_fd_ping_retries }}
524 | {% endif %}
525 | 
526 | # Configurations for Google Compute Engine plugin
527 | {% if elasticsearch_cloud_gce_project_id is defined %}
528 | cloud.gce.project_id: {{ elasticsearch_cloud_gce_project_id }}
529 | {% endif %}
530 | {% if elasticsearch_cloud_gce_zone is defined %}
531 | cloud.gce.zone: {{ elasticsearch_cloud_gce_zone }}
532 | {% endif %}
533 | {% if elasticsearch_discovery_type is defined %}
534 | discovery.type: {{ elasticsearch_discovery_type }}
535 | {% endif %}
536 | {% if elasticsearch_discovery_gce_tags is defined %}
537 | discovery.gce.tags: {{ elasticsearch_discovery_gce_tags }}
538 | {% endif %}
539 | 
540 | 
541 | ################################## Slow Log ##################################
542 | 
543 | # Shard level query and fetch threshold logging.
544 | 
545 | #index.search.slowlog.threshold.query.warn: 10s
546 | {% if elasticsearch_slowlog_threshold_query_warn is defined %}
547 | index.search.slowlog.threshold.query.warn: {{ elasticsearch_slowlog_threshold_query_warn }}
548 | {% endif %}
549 | #index.search.slowlog.threshold.query.info: 5s
550 | {% if elasticsearch_slowlog_threshold_query_info is defined %}
551 | index.search.slowlog.threshold.query.info: {{ elasticsearch_slowlog_threshold_query_info }}
552 | {% endif %}
553 | #index.search.slowlog.threshold.query.debug: 2s
554 | {% if elasticsearch_slowlog_threshold_query_debug is defined %}
555 | index.search.slowlog.threshold.query.debug: {{ elasticsearch_slowlog_threshold_query_debug }}
556 | {% endif %}
557 | #index.search.slowlog.threshold.query.trace: 500ms
558 | {% if elasticsearch_slowlog_threshold_query_trace is defined %}
559 | index.search.slowlog.threshold.query.trace: {{ elasticsearch_slowlog_threshold_query_trace }}
560 | {% endif %}
561 | 
562 | #index.search.slowlog.threshold.fetch.warn: 1s
563 | {% if elasticsearch_slowlog_threshold_fetch_warn is defined %}
564 | index.search.slowlog.threshold.fetch.warn: {{ elasticsearch_slowlog_threshold_fetch_warn }}
565 | {% endif %}
566 | #index.search.slowlog.threshold.fetch.info: 800ms
567 | {% if elasticsearch_slowlog_threshold_fetch_info is defined %}
568 | index.search.slowlog.threshold.fetch.info: {{ elasticsearch_slowlog_threshold_fetch_info }}
569 | {% endif %}
570 | #index.search.slowlog.threshold.fetch.debug: 500ms
571 | {% if elasticsearch_slowlog_threshold_fetch_debug is defined %}
572 | index.search.slowlog.threshold.fetch.debug: {{ elasticsearch_slowlog_threshold_fetch_debug }}
573 | {% endif %}
574 | #index.search.slowlog.threshold.fetch.trace: 200ms
575 | {% if elasticsearch_slowlog_threshold_fetch_trace is defined %}
576 | index.search.slowlog.threshold.fetch.trace: {{ elasticsearch_slowlog_threshold_fetch_trace }}
577 | {% endif %}
578 | 
579 | #index.indexing.slowlog.threshold.index.warn: 10s
580 | {% if elasticsearch_slowlog_threshold_index_warn is defined %}
581 | index.indexing.slowlog.threshold.index.warn: {{ elasticsearch_slowlog_threshold_index_warn }}
582 | {% endif %}
583 | #index.indexing.slowlog.threshold.index.info: 5s
584 | {% if elasticsearch_slowlog_threshold_index_info is defined %}
585 | index.indexing.slowlog.threshold.index.info: {{ elasticsearch_slowlog_threshold_index_info }}
586 | {% endif %}
587 | #index.indexing.slowlog.threshold.index.debug: 2s
588 | {% if elasticsearch_slowlog_threshold_index_debug is defined %}
589 | index.indexing.slowlog.threshold.index.debug: {{ elasticsearch_slowlog_threshold_index_debug }}
590 | {% endif %}
591 | #index.indexing.slowlog.threshold.index.trace: 500ms
592 | {% if elasticsearch_slowlog_threshold_index_trace is defined %}
593 | index.indexing.slowlog.threshold.index.trace: {{ elasticsearch_slowlog_threshold_index_trace }}
594 | {% endif %}
595 | 
596 | ################################## GC Logging ################################
597 | 
598 | #monitor.jvm.gc.ParNew.warn: 1000ms
599 | {% if elasticsearch_gc_par_new_warn is defined %}
600 | monitor.jvm.gc.ParNew.warn: {{ elasticsearch_gc_par_new_warn }}
601 | {% endif %}
602 | #monitor.jvm.gc.ParNew.info: 700ms
603 | {% if elasticsearch_gc_par_new_info is defined %}
604 | monitor.jvm.gc.ParNew.info: {{ elasticsearch_gc_par_new_info }}
605 | {% endif %}
606 | #monitor.jvm.gc.ParNew.debug: 400ms
607 | {% if elasticsearch_gc_par_new_debug is defined %}
608 | monitor.jvm.gc.ParNew.debug: {{ elasticsearch_gc_par_new_debug }}
609 | {% endif %}
610 | 
611 | #monitor.jvm.gc.ConcurrentMarkSweep.warn: 10s
612 | {% if elasticsearch_gc_soncurrent_mark_sweep_warn is defined %}
613 | monitor.jvm.gc.ConcurrentMarkSweep.warn: {{ elasticsearch_gc_soncurrent_mark_sweep_warn }}
614 | {% endif %}
615 | #monitor.jvm.gc.ConcurrentMarkSweep.info: 5s
616 | {% if elasticsearch_gc_soncurrent_mark_sweep_info is defined %}
617 | monitor.jvm.gc.ConcurrentMarkSweep.info: {{ elasticsearch_gc_soncurrent_mark_sweep_info }}
618 | {% endif %}
619 | #monitor.jvm.gc.ConcurrentMarkSweep.debug: 2s
620 | {% if elasticsearch_gc_soncurrent_mark_sweep_debug is defined %}
621 | monitor.jvm.gc.ConcurrentMarkSweep.debug: {{ elasticsearch_gc_soncurrent_mark_sweep_debug }}
622 | {% endif %}
623 | 
624 | ################################### Varia #####################################
625 | 
626 | {% if elasticsearch_plugin_marvel_version is defined %}
627 | 
628 | {% if elasticsearch_plugin_marvel_agent_exporter_es_index_timeformat is defined %}
629 | marvel.agent.exporter.es.index.timeformat: {{ elasticsearch_plugin_marvel_agent_exporter_es_index_timeformat }}
630 | {% endif %}
631 | {% if elasticsearch_plugin_marvel_agent_interval is defined %}
632 | marvel.agent.interval: {{ elasticsearch_plugin_marvel_agent_interval }}
633 | {% endif %}
634 | {% if elasticsearch_plugin_marvel_agent_indices is defined %}
635 | marvel.agent.indices: {{ elasticsearch_plugin_marvel_agent_indices }}
636 | {% endif %}
637 | {% if elasticsearch_plugin_marvel_agent_exporter_es_hosts is defined %}
638 | marvel.agent.exporter.es.hosts: {{ elasticsearch_plugin_marvel_agent_exporter_es_hosts }}
639 | {% endif %}
640 | {% if elasticsearch_plugin_marvel_agent_enabled is defined %}
641 | marvel.agent.enabled: {{ elasticsearch_plugin_marvel_agent_enabled }}
642 | {% endif %}
643 | 
644 | {% endif %}
645 | 
646 | {% if elasticsearch_misc_auto_create_index is defined %}
647 | action.auto_create_index: {{ elasticsearch_misc_auto_create_index }}
648 | {% endif %}
649 | {% if elasticsearch_misc_disable_delete_all_indices is defined %}
650 | action.disable_delete_all_indices: {{ elasticsearch_misc_disable_delete_all_indices }}
651 | {% endif %}
652 | {% if elasticsearch_thread_pools is defined %}
653 | {% for threadPoolSetting in elasticsearch_thread_pools %}
654 | {{ threadPoolSetting }}
655 | {% endfor %}
656 | {% endif %}
657 | 
658 | 
659 | {% if elasticsearch_indices_cache_filter_size is defined %}
660 | indices.cache.filter.size: {{ elasticsearch_indices_cache_filter_size }}
661 | {% endif %}
662 | 
663 | ################################### Dynamic Scripting #####################################
664 | 
665 | {% if elasticsearch_script_disable_dynamic is defined %}
666 | script.disable_dynamic: {{ elasticsearch_script_disable_dynamic }}
667 | {% endif %}
668 | 
669 | ################################### CORS Settings #####################################
670 | 
671 | {% if elasticsearch_http_cors_enabled is defined %}
672 | http.cors.enabled: {{ elasticsearch_http_cors_enabled }}
673 | {% endif %}
674 | 
675 | {% if elasticsearch_http_cors_allow_origin is defined %}
676 | http.cors.allow-origin: {{ elasticsearch_http_cors_allow_origin }}
677 | {% endif %}
678 | 
679 | {% if elasticsearch_http_cors_allow_credentials is defined%}
680 | http.cors.allow-credentials: {{ elasticsearch_http_cors_allow_credentials }}
681 | {% endif %}
682 | 
683 | {% if elasticsearch_script_groovy_sandbox_enabled is defined %}
684 | script.groovy.sandbox.enabled: {{ elasticsearch_script_groovy_sandbox_enabled }}
685 | {% endif %}
686 | 
687 | 
688 | ################################### Awareness Settings #####################################
689 | 
690 | {% if elasticsearch_cluster_routing_allocation_zone_awareness is defined %}
691 | cluster.routing.allocation.awareness.attributes: zone
692 | node.zone: {{ facter_ec2_placement_availability_zone }}
693 | {% endif %}
694 | 
695 | 
696 | 
697 | {% if elasticsearch_plugin_shield_enabled is defined %}
698 | {% if elasticsearch_plugin_shield_enabled == "false"  %}
699 | 
700 | shield.enabled: false
701 | 
702 | {% else %}
703 | ################################### Shield Settings #####################################
704 | 
705 | {% if elasticsearch_plugin_shield_ssl_keystore_path is defined %}
706 | shield.ssl.keystore.path: {{ elasticsearch_plugin_shield_ssl_keystore_path }}
707 | {% endif %}
708 | 
709 | {% if elasticsearch_plugin_shield_ssl_keystore_password is defined %}
710 | shield.ssl.keystore.password: {{ elasticsearch_plugin_shield_ssl_keystore_password }}
711 | {% endif %}
712 | 
713 | {% if elasticsearch_plugin_shield_ssl_keystore_key_password is defined %}
714 | shield.ssl.keystore.key_password: {{ elasticsearch_plugin_shield_ssl_keystore_key_password }}
715 | {% endif %}
716 | 
717 | {% if elasticsearch_plugin_shield_ssl_hostname_verification is defined %}
718 | shield.ssl.hostname_verification: {{ elasticsearch_plugin_shield_ssl_hostname_verification }}
719 | {% endif %}
720 | 
721 | {% if elasticsearch_plugin_shield_transport_ssl is defined %}
722 | shield.transport.ssl: {{ elasticsearch_plugin_shield_transport_ssl }}
723 | {% endif %}
724 | 
725 | {% if elasticsearch_plugin_shield_http_ssl is defined %}
726 | shield.http.ssl: {{ elasticsearch_plugin_shield_http_ssl }}
727 | {% endif %}
728 | 
729 | {% if elasticsearch_plugin_shield_audit_enabled is defined %}
730 | shield.audit.enabled: {{elasticsearch_plugin_shield_audit_enabled}}
731 | {% endif %}
732 | 
733 | {% if elasticsearch_plugin_shield_realms is defined %}
734 | shield.authc.realms:
735 | {% if elasticsearch_plugin_shield_realms_esusers is defined %}
736 |   esusers:
737 |     type: esusers
738 |     order: {{ elasticsearch_plugin_shield_realms_esusers.order }}
739 |     enabled: {{ elasticsearch_plugin_shield_realms_esusers.enabled }}
740 | {% endif %}
741 | 
742 | {% if elasticsearch_plugin_shield_realms_active_directory is defined %}
743 |   active_directory:
744 |     type: active_directory
745 |     order: {{ elasticsearch_plugin_shield_realms_active_directory.order }}
746 |     enabled: {{ elasticsearch_plugin_shield_realms_active_directory.enabled }}
747 |     domain_name: {{ elasticsearch_plugin_shield_realms_active_directory.domain_name }}
748 |     unmapped_groups_as_roles: {{ elasticsearch_plugin_shield_realms_active_directory.unmapped_groups_as_roles }}
749 |     url: {{ elasticsearch_plugin_shield_realms_active_directory.url }}
750 |     #cache.ttl: 1m
751 | {% endif %}
752 | {% endif %}
753 | {% endif %}
754 | 
755 | 
756 | {% endif %}
757 | 
758 | {% if elasticsearch_plugin_mcf is defined %}
759 | ################################### ManifoldCF Settings #####################################
760 | {% if elasticsearch_plugin_mcf_allow_field_prefix is defined %}
761 | mcf.allow_field_prefix: {{elasticsearch_plugin_mcf_allow_field_prefix}}
762 | {% endif %}
763 | 
764 | {% if elasticsearch_plugin_mcf_deny_field_prefix is defined %}
765 | mcf.deny_field_prefix: {{elasticsearch_plugin_mcf_deny_field_prefix}}
766 | {% endif %}
767 | 
768 | {% endif %}
769 | 
770 | {% if elasticsearch_lang_config_files_hunspell_destination is defined %}
771 | indices.analysis.hunspell.dictionary.location: {{elasticsearch_lang_config_files_hunspell_destination}}
772 | indices.analysis.hunspell.dictionary.ignore_case: {{elasticsearch_lang_hunspell_global_ignore_case}}
773 | {% endif %}


--------------------------------------------------------------------------------