├── .gitignore ├── LICENSE ├── README.md ├── configuration ├── elastic-env.xml ├── elastic-site.xml └── elastic-sysconfig.xml ├── metainfo.xml ├── package ├── scripts │ ├── elastic.py │ ├── elastic_master.py │ ├── elastic_slave.py │ ├── params.py │ ├── properties_config.py │ ├── service_check.py │ ├── slave.py │ └── status_params.py └── templates │ ├── elasticsearch.master.yaml.j2 │ └── elasticsearch.slave.yaml.j2 └── role_command_order.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Phakin Cheangkrachange 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Elasticsearch Service for Ambari 2 | 3 | A custom service for Ambari which allows you to install and manage Elasticsearch via Ambari. This service is not supported by Hortonworks. Futhermore, this service is intended for testing and development and should not be used in a production environment. 4 | 5 | ## Compatibility 6 | 7 | This service has been tested with the following: 8 | 9 | - CentOS 7.x 10 | - Ambari 2.5,2.6 11 | - HDP/HDF 2.x/3.x 12 | - ElasticSearch 6.x 13 | 14 | ## Installation 15 | 16 | To install this service, you need access to the Ambari Server with sudo permissions. 17 | 18 | ``` 19 | VERSION=`hdp-select status hadoop-client | sed 's/hadoop-client - \([0-9]\.[0-9]\).*/\1/'` 20 | sudo git clone https://github.com/teamsoo/elasticsearch-ambari /var/lib/ambari-server/resources/stacks/HDP/$VERSION/services/ELASTICSEARCH 21 | ``` 22 | 23 | If you do not have the ability to use git, you can download the repo archive and extract it to directory shown above. 24 | 25 | After you have installed the service, you need to restart the Ambari Server. 26 | 27 | ``` 28 | sudo service ambari-server restart 29 | ``` 30 | 31 | Once the Ambari Server has been restarted, you should see Elasticsearch as an available service to install from the Add Service screen. 32 | 33 | ## Required property 34 | ``` 35 | zen_discovery_ping_unicast_hosts - FQDN of master and data nodes. seperated by comma eg. master.internal,data1.internal,data2.internal 36 | ``` 37 | 38 | ## License 39 | This project is based on Elasticsearch Ambari Service of 40 | -------------------------------------------------------------------------------- /configuration/elastic-env.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 20 | 21 | 22 | 23 | elastic_user 24 | elasticsearch 25 | USER 26 | The user for Elasticsearch 27 | 28 | 29 | user_group 30 | elasticsearch 31 | The group for Elasticsearch 32 | 33 | 34 | elastic_log_dir 35 | /var/log/elasticsearch 36 | Log directory for elastic 37 | 38 | 39 | elastic_pid_dir 40 | /var/run/elasticsearch 41 | The directory for pid files 42 | 43 | 44 | 45 | 46 | content 47 | This is the jinja template for elastic-env.sh file 48 | 49 | #!/bin/bash 50 | 51 | # Set ELASTICSEARCH specific environment variables here. 52 | 53 | # The java implementation to use. 54 | export JAVA_HOME={{java64_home}} 55 | export PATH=$PATH:$JAVA_HOME/bin 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /configuration/elastic-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 20 | 21 | 22 | 23 | 24 | 25 | cluster_name 26 | elasticsearch 27 | Cluster name identifies your cluster 28 | 29 | 30 | zen_discovery_ping_unicast_hosts 31 | 32 | Unicast discovery list of hosts to act as gossip routers, in comma separated format. 33 | 34 | 35 | index_number_of_shards 36 | 4 37 | Set the number of shards (splits) of an index 38 | 39 | 40 | index_number_of_replicas 41 | 2 42 | Set the number of replicas (additional copies) of an index 43 | 44 | 45 | 46 | path_data 47 | "/opt/lmm/es_data" 48 | Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second" 49 | 50 | 51 | http_cors_enabled 52 | "false" 53 | Enable or disable cross-origin resource sharing, i.e. whether a browser on another origin can do requests to Elasticsearch. Defaults to false. 54 | 55 | string 56 | 57 | 58 | 59 | 60 | transport_tcp_port 61 | 9300-9400 62 | Set a custom port for the node to node communication 63 | 64 | 65 | http_port 66 | 9200-9300 67 | Set a custom port to listen for HTTP traffic 68 | 69 | 70 | 71 | discovery_zen_ping_multicast_enabled 72 | false 73 | master eligible nodes 74 | 75 | 76 | discovery_zen_ping_timeout 77 | 3s 78 | Wait for ping responses for master discovery 79 | 80 | 81 | discovery_zen_fd_ping_interval 82 | 15s 83 | Wait for ping for cluster discovery 84 | 85 | 86 | discovery_zen_fd_ping_timeout 87 | 60s 88 | Wait for ping for cluster discovery 89 | 90 | 91 | discovery_zen_fd_ping_retries 92 | 5 93 | Number of ping retries before blacklisting 94 | 95 | 96 | 97 | gateway_recover_after_data_nodes 98 | 3 99 | Recover as long as this many data or master nodes have joined the cluster. 100 | 101 | 102 | recover_after_time 103 | 15m 104 | recover_after_time 105 | 106 | 107 | expected_data_nodes 108 | 0 109 | expected_data_nodes 110 | 111 | 112 | 113 | index_merge_scheduler_max_thread_count 114 | 5 115 | index.merge.scheduler.max_thread_count 116 | 117 | 118 | indices_memory_index_store_throttle_type 119 | none 120 | index_store_throttle_type 121 | 122 | 123 | index_refresh_interval 124 | 1s 125 | index refresh interval 126 | 127 | 128 | index_translog_flush_threshold_size 129 | 5g 130 | index_translog_flush_threshold_size 131 | 132 | 133 | indices_memory_index_buffer_size 134 | 10% 135 | Percentage of heap used for write buffers 136 | 137 | 138 | bootstrap_mlockall 139 | true 140 | The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out 141 | 142 | 143 | threadpool_bulk_queue_size 144 | 3000 145 | It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request 146 | 147 | 148 | threadpool_index_queue_size 149 | 1000 150 | It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request 151 | 152 | 153 | indices_cluster_send_refresh_mapping 154 | false 155 | In order to make the index request more efficient, we have set this property on our data nodes 156 | 157 | 158 | indices_fielddata_cache_size 159 | 25% 160 | You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index 161 | 162 | 163 | cluster_routing_allocation_disk_watermark_high 164 | 0.99 165 | Property used when multiple drives are used to understand max thresholds 166 | 167 | 168 | cluster_routing_allocation_disk_threshold_enabled 169 | true 170 | Property used when multiple drives are used to understand if thresholding is active 171 | 172 | 173 | cluster_routing_allocation_disk_watermark_low 174 | .97 175 | Property used when multiple drives are used to understand min thresholds 176 | 177 | 178 | cluster_routing_allocation_node_concurrent_recoveries 179 | 4 180 | Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart 181 | 182 | 183 | network_host 184 | _lo_,_eth0_ 185 | Network interface(s) will bind to. 186 | 187 | 188 | -------------------------------------------------------------------------------- /configuration/elastic-sysconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 20 | 21 | 22 | 23 | elastic_home 24 | /usr/share/elasticsearch/ 25 | Elasticsearch Home Directory 26 | 27 | 28 | data_dir 29 | /var/lib/elasticsearch/ 30 | Elasticsearch Data Directory 31 | 32 | 33 | work_dir 34 | /tmp/elasticsearch/ 35 | Elasticsearch Work Directory 36 | 37 | 38 | conf_dir 39 | /etc/elasticsearch/ 40 | Elasticsearch Configuration Directory 41 | 42 | 43 | heap_size 44 | 128m 45 | Heap size 46 | 47 | 48 | max_open_files 49 | 65535 50 | Maximum number of open files 51 | 52 | 53 | max_map_count 54 | 262144 55 | Maximum number of memory map areas for process 56 | 57 | 58 | 59 | 60 | content 61 | This is the jinja template for elastic-env.sh file 62 | 63 | # Directory where the Elasticsearch binary distribution resides 64 | ES_HOME={{elastic_home}} 65 | 66 | # Heap Size (defaults to 256m min, 1g max) 67 | ES_HEAP_SIZE={{heap_size}} 68 | 69 | # Maximum number of open files 70 | MAX_OPEN_FILES={{max_open_files}} 71 | 72 | # Maximum number of VMA (Virtual Memory Areas) a process can own 73 | MAX_MAP_COUNT={{max_map_count}} 74 | 75 | # Elasticsearch log directory 76 | LOG_DIR={{log_dir}} 77 | 78 | # Elasticsearch data directory 79 | DATA_DIR={{data_dir}} 80 | 81 | # Elasticsearch work directory 82 | WORK_DIR={{work_dir}} 83 | 84 | # Elasticsearch conf directory 85 | CONF_DIR={{conf_dir}} 86 | 87 | # User to run as, change this to a specific elasticsearch user if possible 88 | # Also make sure, this user can write into the log directories in case you change them 89 | # This setting only works for the init script, but has to be configured separately for systemd startup 90 | ES_USER={{elastic_user}} 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 2.0 20 | 21 | 22 | AMBARI_ELASTICSEARCH 23 | Elasticsearch 24 | Indexing and Search 25 | 6.0 26 | 27 | 28 | ES_MASTER 29 | Elasticsearch Master 30 | MASTER 31 | 1+ 32 | 33 | 34 | PYTHON 35 | 600 36 | 37 | 38 | 39 | ES_SLAVE 40 | Elasticsearch Data Node 41 | SLAVE 42 | 3+ 43 | 44 | 45 | PYTHON 46 | 600 47 | 48 | 49 | 50 | 51 | 52 | any 53 | 54 | 55 | elasticsearch-6.0 56 | 57 | 58 | 59 | 60 | 61 | 62 | PYTHON 63 | 300 64 | 65 | 66 | elastic-env 67 | elastic-site 68 | elastic-sysconfig 69 | 70 | true 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /package/scripts/elastic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | """ 20 | 21 | from resource_management.core.resources.system import Directory 22 | from resource_management.core.resources.system import File 23 | from resource_management.core.source import InlineTemplate 24 | from resource_management.core.source import Template 25 | 26 | 27 | def elastic(): 28 | print "INSIDE THE %s" % __file__ 29 | import params 30 | 31 | params.path_data = params.path_data.replace('"', '') 32 | data_path = params.path_data.replace(' ', '').split(',') 33 | data_path[:] = [x.replace('"', '') for x in data_path] 34 | 35 | directories = [params.log_dir, params.pid_dir, params.conf_dir] 36 | directories = directories + data_path 37 | 38 | Directory(directories, 39 | create_parents=True, 40 | # recursive=True, 41 | mode=0755, 42 | owner=params.elastic_user, 43 | group=params.elastic_user 44 | ) 45 | 46 | print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir) 47 | File("{0}/elastic-env.sh".format(params.conf_dir), 48 | owner=params.elastic_user, 49 | content=InlineTemplate(params.elastic_env_sh_template) 50 | ) 51 | 52 | configurations = params.config['configurations']['elastic-site'] 53 | 54 | print "Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir) 55 | File("{0}/elasticsearch.yml".format(params.conf_dir), 56 | content=Template( 57 | "elasticsearch.master.yaml.j2", 58 | configurations=configurations), 59 | owner=params.elastic_user, 60 | group=params.elastic_user 61 | ) 62 | 63 | print "Master sysconfig: /etc/sysconfig/elasticsearch" 64 | File(format("/etc/sysconfig/elasticsearch"), 65 | owner="root", 66 | group="root", 67 | content=InlineTemplate(params.sysconfig_template) 68 | ) 69 | -------------------------------------------------------------------------------- /package/scripts/elastic_master.py: -------------------------------------------------------------------------------- 1 | """ 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | """ 19 | 20 | from resource_management.core.resources.system import Execute 21 | from resource_management.libraries.script import Script 22 | 23 | from elastic import elastic 24 | 25 | 26 | class Elasticsearch(Script): 27 | def install(self, env): 28 | import params 29 | env.set_params(params) 30 | 31 | print 'Install the Master' 32 | Execute('rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch') 33 | Execute(format("echo \"[elasticsearch-2.x]\n" 34 | "name=Elasticsearch repository for 6.x packages\n" 35 | "baseurl=https://artifacts.elastic.co/packages/6.x/yum\n" 36 | "gpgcheck=1\n" 37 | "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch\n" 38 | "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")) 39 | 40 | self.install_packages(env) 41 | 42 | def configure(self, env, upgrade_type=None, config_dir=None): 43 | import params 44 | env.set_params(params) 45 | 46 | elastic() 47 | 48 | def stop(self, env, upgrade_type=None): 49 | import params 50 | env.set_params(params) 51 | stop_cmd = format("service elasticsearch stop") 52 | print 'Stop the Master' 53 | Execute(stop_cmd) 54 | 55 | def start(self, env, upgrade_type=None): 56 | import params 57 | env.set_params(params) 58 | 59 | self.configure(env) 60 | start_cmd = format("service elasticsearch start") 61 | print 'Start the Master' 62 | Execute(start_cmd) 63 | 64 | def status(self, env): 65 | import params 66 | env.set_params(params) 67 | status_cmd = format("service elasticsearch status") 68 | print 'Status of the Master' 69 | Execute(status_cmd) 70 | 71 | def restart(self, env): 72 | import params 73 | env.set_params(params) 74 | self.configure(env) 75 | restart_cmd = format("service elasticsearch restart") 76 | print 'Restarting the Master' 77 | Execute(restart_cmd) 78 | 79 | 80 | if __name__ == "__main__": 81 | Elasticsearch().execute() 82 | -------------------------------------------------------------------------------- /package/scripts/elastic_slave.py: -------------------------------------------------------------------------------- 1 | """ 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | """ 19 | 20 | from resource_management.core.resources.system import Execute 21 | from resource_management.libraries.script import Script 22 | 23 | from slave import slave 24 | 25 | 26 | class Elasticsearch(Script): 27 | def install(self, env): 28 | import params 29 | env.set_params(params) 30 | print 'Install the Slave' 31 | Execute('rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch') 32 | Execute("echo \"[elasticsearch-2.x]\n" 33 | "name=Elasticsearch repository for 6.x packages\n" 34 | "baseurl=https://artifacts.elastic.co/packages/6.x/yum\n" 35 | "gpgcheck=1\n" 36 | "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch\n" 37 | "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") 38 | self.install_packages(env) 39 | 40 | def configure(self, env, upgrade_type=None, config_dir=None): 41 | import params 42 | env.set_params(params) 43 | slave() 44 | 45 | def stop(self, env, upgrade_type=None): 46 | import params 47 | env.set_params(params) 48 | stop_cmd = format("service elasticsearch stop") 49 | print 'Stop the Slave' 50 | Execute(stop_cmd) 51 | 52 | def start(self, env, upgrade_type=None): 53 | import params 54 | env.set_params(params) 55 | self.configure(env) 56 | start_cmd = format("service elasticsearch start") 57 | print 'Start the Slave' 58 | Execute(start_cmd) 59 | 60 | def status(self, env): 61 | import params 62 | env.set_params(params) 63 | status_cmd = format("service elasticsearch status") 64 | print 'Status of the Slave' 65 | Execute(status_cmd) 66 | 67 | def restart(self, env): 68 | import params 69 | env.set_params(params) 70 | self.configure(env) 71 | restart_cmd = format("service elasticsearch restart") 72 | print 'Restarting the Slave' 73 | Execute(restart_cmd) 74 | 75 | 76 | if __name__ == "__main__": 77 | Elasticsearch().execute() 78 | -------------------------------------------------------------------------------- /package/scripts/params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | """ 20 | 21 | from resource_management.libraries.script import Script 22 | 23 | # server configurations 24 | config = Script.get_config() 25 | 26 | elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] 27 | data_dir = config['configurations']['elastic-sysconfig']['data_dir'] 28 | work_dir = config['configurations']['elastic-sysconfig']['work_dir'] 29 | conf_dir = config['configurations']['elastic-sysconfig']['conf_dir'] 30 | heap_size = config['configurations']['elastic-sysconfig']['heap_size'] 31 | max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] 32 | max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] 33 | 34 | elastic_user = config['configurations']['elastic-env']['elastic_user'] 35 | user_group = config['configurations']['elastic-env']['user_group'] 36 | log_dir = config['configurations']['elastic-env']['elastic_log_dir'] 37 | pid_dir = '/var/run/elasticsearch' 38 | pid_file = '/var/run/elasticsearch/elasticsearch.pid' 39 | hostname = config['hostname'] 40 | java64_home = config['hostLevelParams']['java_home'] 41 | elastic_env_sh_template = config['configurations']['elastic-env']['content'] 42 | sysconfig_template = config['configurations']['elastic-sysconfig']['content'] 43 | 44 | cluster_name = config['configurations']['elastic-site']['cluster_name'] 45 | zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts'] 46 | 47 | path_data = config['configurations']['elastic-site']['path_data'] 48 | http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled'] 49 | http_port = config['configurations']['elastic-site']['http_port'] 50 | transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port'] 51 | 52 | recover_after_time = config['configurations']['elastic-site']['recover_after_time'] 53 | gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] 54 | expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] 55 | discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled'] 56 | index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] 57 | index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] 58 | index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] 59 | indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type'] 60 | index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] 61 | index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] 62 | indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] 63 | bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall'] 64 | threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] 65 | cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] 66 | cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] 67 | cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'] 68 | cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high'] 69 | indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size'] 70 | indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'] 71 | threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size'] 72 | 73 | discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout'] 74 | discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval'] 75 | discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout'] 76 | discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] 77 | 78 | network_host = config['configurations']['elastic-site']['network_host'] 79 | -------------------------------------------------------------------------------- /package/scripts/properties_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | """ 20 | 21 | from resource_management.core.resources.system import File 22 | from resource_management.core.source import InlineTemplate 23 | 24 | 25 | def properties_inline_template(configurations): 26 | return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }} 27 | {% endfor %}''', configurations_dict=configurations) 28 | 29 | 30 | def properties_config(filename, configurations=None, conf_dir=None, 31 | mode=None, owner=None, group=None, brokerid=None): 32 | config_content = properties_inline_template(configurations) 33 | File(format("{conf_dir}/{filename}"), content=config_content, owner=owner, 34 | group=group, mode=mode) 35 | -------------------------------------------------------------------------------- /package/scripts/service_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | """ 20 | from __future__ import print_function 21 | 22 | import subprocess 23 | import sys 24 | 25 | from resource_management.core.resources.system import Execute 26 | from resource_management.libraries.script import Script 27 | 28 | 29 | class ServiceCheck(Script): 30 | def service_check(self, env): 31 | import params 32 | env.set_params(params) 33 | 34 | doc = '{"name": "Ambari Smoke test"}' 35 | index = "ambari_smoke_test" 36 | 37 | print("Running Elastic search service check", file=sys.stdout) 38 | 39 | # Make sure the service is actually up. We can live without everything allocated. 40 | # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can 41 | # also hit ES before things are green. 42 | host = "localhost:9200" 43 | Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host, 44 | logoutput=True, 45 | tries=6, 46 | try_sleep=20 47 | ) 48 | 49 | # Put a document into a new index. 50 | 51 | Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True) 52 | 53 | # Retrieve the document. Use subprocess because we actually need the results here. 54 | cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index) 55 | proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 56 | (stdout, stderr) = proc.communicate() 57 | response_retrieve = stdout 58 | print("Retrieval response is: %s" % response_retrieve) 59 | expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \ 60 | % (index, doc) 61 | 62 | # Delete the index 63 | cmd_delete = "curl -XDELETE '%s/%s'" % (host, index) 64 | proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 65 | (stdout, stderr) = proc.communicate() 66 | response_delete = stdout 67 | print("Delete index response is: %s" % response_retrieve) 68 | expected_delete = '{"acknowledged":true}' 69 | 70 | if (expected_retrieve == response_retrieve) and (expected_delete == response_delete): 71 | print("Smoke test able to communicate with Elasticsearch") 72 | else: 73 | print("Elasticsearch service unable to retrieve document.") 74 | sys.exit(1) 75 | 76 | exit(0) 77 | 78 | 79 | if __name__ == "__main__": 80 | ServiceCheck().execute() 81 | -------------------------------------------------------------------------------- /package/scripts/slave.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | """ 20 | 21 | from resource_management.core.resources.system import Directory 22 | from resource_management.core.resources.system import File 23 | from resource_management.core.source import InlineTemplate 24 | from resource_management.core.source import Template 25 | 26 | 27 | def slave(): 28 | import params 29 | 30 | params.path_data = params.path_data.replace('"', '') 31 | data_path = params.path_data.replace(' ', '').split(',') 32 | data_path[:] = [x.replace('"', '') for x in data_path] 33 | 34 | directories = [params.log_dir, params.pid_dir, params.conf_dir] 35 | directories = directories + data_path 36 | 37 | Directory(directories, 38 | create_parents=True, 39 | mode=0755, 40 | owner=params.elastic_user, 41 | group=params.elastic_user, 42 | cd_access="a" 43 | ) 44 | 45 | File("{0}/elastic-env.sh".format(params.conf_dir), 46 | owner=params.elastic_user, 47 | content=InlineTemplate(params.elastic_env_sh_template) 48 | ) 49 | 50 | configurations = params.config['configurations']['elastic-site'] 51 | 52 | File("{0}/elasticsearch.yml".format(params.conf_dir), 53 | content=Template( 54 | "elasticsearch.slave.yaml.j2", 55 | configurations=configurations), 56 | owner=params.elastic_user, 57 | group=params.elastic_user 58 | ) 59 | 60 | print "Master sysconfig: /etc/sysconfig/elasticsearch" 61 | File(format("/etc/sysconfig/elasticsearch"), 62 | owner="root", 63 | group="root", 64 | content=InlineTemplate(params.sysconfig_template) 65 | ) 66 | -------------------------------------------------------------------------------- /package/scripts/status_params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | """ 20 | 21 | from resource_management.libraries.script import Script 22 | 23 | config = Script.get_config() 24 | 25 | elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] 26 | elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid") 27 | -------------------------------------------------------------------------------- /package/templates/elasticsearch.master.yaml.j2: -------------------------------------------------------------------------------- 1 | {# 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | #} 18 | 19 | cluster: 20 | name: {{cluster_name}} 21 | routing: 22 | allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} 23 | allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} 24 | allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} 25 | allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} 26 | 27 | discovery: 28 | zen: 29 | ping: 30 | multicast: 31 | enabled: {{discovery_zen_ping_multicast_enabled}} 32 | unicast: 33 | hosts: "{{zen_discovery_ping_unicast_hosts}}" 34 | 35 | node: 36 | data: false 37 | master: true 38 | name: {{hostname}} 39 | path: 40 | data: {{path_data}} 41 | 42 | http.cors.enabled: {{http_cors_enabled}} 43 | 44 | port: {{http_port}} 45 | 46 | transport: 47 | tcp: 48 | port: {{transport_tcp_port}} 49 | 50 | gateway: 51 | recover_after_data_nodes: {{gateway_recover_after_data_nodes}} 52 | recover_after_time: {{recover_after_time}} 53 | expected_data_nodes: {{expected_data_nodes}} 54 | 55 | index: 56 | number_of_shards: {{index_number_of_shards}} 57 | merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} 58 | translog.flush_threshold_size: {{index_translog_flush_threshold_size}} 59 | refresh_interval: {{index_refresh_interval}} 60 | number_of_replicas: {{index_number_of_replicas}} 61 | 62 | indices: 63 | memory: 64 | index_buffer_size: {{indices_memory_index_buffer_size}} 65 | store.throttle.type: {{indices_memory_index_store_throttle_type}} 66 | fielddata: 67 | cache.size: {{indices_fielddata_cache_size}} 68 | cluster: 69 | send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} 70 | 71 | bootstrap.mlockall: {{bootstrap_mlockall}} 72 | 73 | threadpool: 74 | bulk: 75 | queue_size: {{threadpool_bulk_queue_size}} 76 | index: 77 | queue_size: {{threadpool_index_queue_size}} 78 | 79 | discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} 80 | discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} 81 | discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} 82 | discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} 83 | 84 | network.host: {{network_host}} -------------------------------------------------------------------------------- /package/templates/elasticsearch.slave.yaml.j2: -------------------------------------------------------------------------------- 1 | {# 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | #} 18 | 19 | cluster: 20 | name: {{cluster_name}} 21 | routing: 22 | allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} 23 | allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} 24 | allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} 25 | allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} 26 | 27 | discovery: 28 | zen: 29 | ping: 30 | multicast: 31 | enabled: {{discovery_zen_ping_multicast_enabled}} 32 | unicast: 33 | hosts: "{{zen_discovery_ping_unicast_hosts}}" 34 | 35 | node: 36 | data: true 37 | master: false 38 | name: {{hostname}} 39 | path: 40 | data: {{path_data}} 41 | 42 | http.cors.enabled: {{http_cors_enabled}} 43 | 44 | port: {{http_port}} 45 | 46 | transport: 47 | tcp: 48 | port: {{transport_tcp_port}} 49 | 50 | gateway: 51 | recover_after_data_nodes: {{gateway_recover_after_data_nodes}} 52 | recover_after_time: {{recover_after_time}} 53 | expected_data_nodes: {{expected_data_nodes}} 54 | 55 | index: 56 | number_of_shards: {{index_number_of_shards}} 57 | merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} 58 | translog.flush_threshold_size: {{index_translog_flush_threshold_size}} 59 | refresh_interval: {{index_refresh_interval}} 60 | number_of_replicas: {{index_number_of_replicas}} 61 | 62 | indices: 63 | memory: 64 | index_buffer_size: {{indices_memory_index_buffer_size}} 65 | store.throttle.type: {{indices_memory_index_store_throttle_type}} 66 | fielddata: 67 | cache.size: {{indices_fielddata_cache_size}} 68 | cluster: 69 | send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} 70 | 71 | bootstrap.mlockall: {{bootstrap_mlockall}} 72 | 73 | threadpool: 74 | bulk: 75 | queue_size: {{threadpool_bulk_queue_size}} 76 | index: 77 | queue_size: {{threadpool_index_queue_size}} 78 | 79 | discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} 80 | discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} 81 | discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} 82 | discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} 83 | 84 | network.host: {{network_host}} -------------------------------------------------------------------------------- /role_command_order.json: -------------------------------------------------------------------------------- 1 | { 2 | "_comment" : "Record format:", 3 | "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", 4 | "general_deps" : { 5 | "_comment" : "dependencies for all cases", 6 | "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"] 7 | } 8 | } 9 | --------------------------------------------------------------------------------