├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── ambari_configs.py ├── common.py ├── config_llap.py └── layout_rpt.py /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ 2 | .idea 3 | *.iml 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hive LLAP Calculator 2 | 3 | Memory / Configuration Calculator for Hive LLAP. 4 | 5 | ## Change Log 6 | 7 | | Date | Change | Issues Link | 8 | | :------ | :----- | :--- | 9 | | 2020-07-17 | Moved up 'pmem' check to recommendation for all configs. | [#19](https://github.com/dstreev/hive_llap_calculator/issues/19) | 10 | | 2019-11-21 | Added ability to define separate Concurrency Queue | [#17](https://github.com/dstreev/hive_llap_calculator/issues/17) | 11 | | 2019-11-15 | Displays, defaults and auto guide. Reverted Issue #14 after more research | [#16](https://github.com/dstreev/hive_llap_calculator/issues/16) | 12 | | 2019-11-07 | Change threshold to property for: `hive.llap.daemon.memory.per.instance.mb`. Default Percent for Executors increased to 100 | [#14](https://github.com/dstreev/hive_llap_calculator/issues/14) [#15](https://github.com/dstreev/hive_llap_calculator/issues/15) | 13 | | 2019-10-30 | Added Rules to Save Action | | 14 | | 2019-10-29 | Added Rule to check imbalance of cores and memory | | 15 | | 2019-10-29 | Added support for pulling current values from Ambari Blueprint | [#3](https://github.com/dstreev/hive_llap_calculator/issues/3) | 16 | | 2019-10-16 | Added Safety Valve for Daemons over 256Gb. Grace space to help reduce YARN container KILLs | | 17 | | 2019-10-09 | Added Issue/Solution Description for Daemon Sizes over 256Gb | | 18 | | 2019-09-19 | Added integration with 'working' values on initial collection with ambari integration. Allow mods to LLAP Min Task Allocation. Ability to control output display columns (see m:mode). Save to File. Initial Error Printouts | [#5](https://github.com/dstreev/hive_llap_calculator/issues/5) [#6](https://github.com/dstreev/hive_llap_calculator/issues/6) [#8](https://github.com/dstreev/hive_llap_calculator/issues/8) [#11](https://github.com/dstreev/hive_llap_calculator/issues/11) [#12](https://github.com/dstreev/hive_llap_calculator/issues/12)| 19 | | 2019-09-18 | Add Min check for LLAP Daemon Heap. This value must be no less then 4gb * Number of Executors configured | [#1](https://github.com/dstreev/hive_llap_calculator/issues/1) | 20 | |2019-09-17 | Changed the method for calculating total memory and resources for LLAP in the Cluster. We use to use a 'percentage' of the cluster as a starting point for calculating the number of LLAP nodes. This was a little cryptic. The new method asks how many 'nodes' you want to run LLAP on. The assumption now is that LLAP will consume the whole node. So memory calculations start from the available YARN memory on a node and not a 'percentage' of that memory. | [#2](https://github.com/dstreev/hive_llap_calculator/issues/2) [#4](https://github.com/dstreev/hive_llap_calculator/issues/4) | 21 | 22 | ## Assumptions 23 | 24 | - Tested on: 25 | - Mac OS X iTerm and Terminal 26 | - CentOS 7 bash 27 | - Python 2.7+ 28 | - Calculator for Hive 3+ LLAP (HDP 3+) (May work for Hive 2 LLAP, but not tested) 29 | 30 | ## Limitations 31 | 32 | This tool does NOT support 'heterogenous' environment calculations. Calculations are based on a 'homogenous' compute configuration. If you have a 'heterogenous' compute environment, limit the calculations to a subset of 'heterogenous' nodes where LLAP daemons will run. In this scenario, the calculations for Queue sizes and overall footprint percentages will NOT be accurate. Please use the numbers provided as a base to calculate those values for the full cluster. 33 | 34 | ## Directions 35 | 36 | Run 37 | 38 | ``` 39 | ./config_llap.py 40 | ``` 41 | 42 | Follow the prompts. 43 | 44 | 'Ambari Config List' will produce commands that can be run with [Ambari's Command Interface](ambari_configs.py) 45 | 46 | You'll need to create a few environment variables to support running the Ambari command outputs: 47 | - AMBARI_HOST 48 | - AMBARI_PORT 49 | - CLUSTER_NAME 50 | 51 | Use a 'credentials' file for User/Password information for Ambari. Create a file in your ${HOME} directory called `.ambari-credentials` 52 | 53 | The contents should be 2 lines. First line is the user, second is the password. NOTE: This user should be a 'local' account. You may have issues with 'remote' accounts and/or SSO accounts. 54 | 55 | ``` 56 | david 57 | password123 58 | ``` 59 | 60 | ## Help 61 | ``` 62 | ./config_llap.py --help 63 | ``` 64 | ``` 65 | Usage: config_llap.py [options] 66 | 67 | Options: 68 | -h, --help show this help message and exit 69 | -t PORT, --port=PORT Optional port number for Ambari server. Default is 70 | '8080'. Provide empty string to not use port. 71 | -s PROTOCOL, --protocol=PROTOCOL 72 | Optional support of SSL. Default protocol is 'http' 73 | --unsafe Skip SSL certificate verification. 74 | -l HOST, --host=HOST Server external host name 75 | -n CLUSTER, --cluster=CLUSTER 76 | Name given to cluster. Ex: 'c1' 77 | -v VERSION_NOTE, --version-note=VERSION_NOTE 78 | Version change notes which will help to know what has 79 | been changed in this config. This value is optional 80 | and is used for actions and . 81 | -w WORKERS, --workers=WORKERS 82 | How many worker nodes in the cluster? 83 | -m MEMORY, --memory=MEMORY 84 | How much memory does each worker node have (GB)? 85 | -b AMBARI_BLUEPRINT, --ambari-blueprint=AMBARI_BLUEPRINT 86 | Use an Ambari Blueprint to pull configs. 87 | 88 | To specify credentials please use "-e" OR "-u" and "-p'": 89 | -u USER, --user=USER 90 | Optional user ID to use for authentication. Default is 91 | 'admin' 92 | -p PASSWORD, --password=PASSWORD 93 | Optional password to use for authentication. Default 94 | is 'admin' 95 | -e CREDENTIALS_FILE, --credentials-file=CREDENTIALS_FILE 96 | Optional file with user credentials separated by new 97 | line. 98 | ``` 99 | 100 | ## Ambari Integration 101 | You can pull most of the current configurations from Ambari by supplying the following parameters 102 | - [--host, --port, --credentials-file, --cluster] 103 | 104 | If you supply details of the cluster size with 105 | 106 | - [--workers, --memory] 107 | 108 | the system will estimate additional 'totals' for the current environment. Which you can use for validation. 109 | 110 | Example: 111 | ``` 112 | # Connecting to an Ambari Server basic configuration 113 | ./config_llap.py --host --port --cluster --credentials-file credentials.txt --workers 50 --memory 256 114 | 115 | 116 | # Connecting to an Ambari Server with a Self-Signed SSL Cert 117 | ./config_llap.py --host --port --cluster --credentials-file credentials.txt -s https --unsafe --workers 50 --memory 256 118 | ``` 119 | 120 | ## Notes 121 | 122 | [Ambari-Config](./ambari_configs.py) is a copy of the 'configs.py' resource for Ambari-Server found at `/var/lib/ambari-server/resources/scripts` . The version here was pulled from Ambari 2.7.3.0. -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dstreev/hive_llap_calculator/5630d55a8bda182996388dc976ee97e2b6cb58b2/__init__.py -------------------------------------------------------------------------------- /ambari_configs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Licensed to the Apache Software Foundation (ASF) under one 5 | or more contributor license agreements. See the NOTICE file 6 | distributed with this work for additional information 7 | regarding copyright ownership. The ASF licenses this file 8 | to you under the Apache License, Version 2.0 (the 9 | "License"); you may not use this file except in compliance 10 | with the License. You may obtain a copy of the License at 11 | 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | ''' 20 | 21 | import optparse 22 | from optparse import OptionGroup 23 | import sys 24 | import urllib2, ssl 25 | import time 26 | import json 27 | import base64 28 | import xml 29 | import xml.etree.ElementTree as ET 30 | import os 31 | import logging 32 | 33 | logger = logging.getLogger('AmbariConfig') 34 | 35 | HTTP_PROTOCOL = 'http' 36 | HTTPS_PROTOCOL = 'https' 37 | 38 | SET_ACTION = 'set' 39 | GET_ACTION = 'get' 40 | DELETE_ACTION = 'delete' 41 | 42 | GET_REQUEST_TYPE = 'GET' 43 | PUT_REQUEST_TYPE = 'PUT' 44 | 45 | # JSON Keywords 46 | PROPERTIES = 'properties' 47 | ATTRIBUTES = 'properties_attributes' 48 | CLUSTERS = 'Clusters' 49 | DESIRED_CONFIGS = 'desired_configs' 50 | SERVICE_CONFIG_NOTE = 'service_config_version_note' 51 | TYPE = 'type' 52 | TAG = 'tag' 53 | ITEMS = 'items' 54 | TAG_PREFIX = 'version' 55 | 56 | CLUSTERS_URL = '/api/v1/clusters/{0}' 57 | DESIRED_CONFIGS_URL = CLUSTERS_URL + '?fields=Clusters/desired_configs' 58 | CONFIGURATION_URL = CLUSTERS_URL + '/configurations?type={1}&tag={2}' 59 | 60 | FILE_FORMAT = \ 61 | """ 62 | "properties": { 63 | "key1": "value1" 64 | "key2": "value2" 65 | }, 66 | "properties_attributes": { 67 | "attribute": { 68 | "key1": "value1" 69 | "key2": "value2" 70 | } 71 | } 72 | """ 73 | 74 | class UsageException(Exception): 75 | pass 76 | 77 | 78 | def api_accessor(host, login, password, protocol, port, unsafe=None): 79 | def do_request(api_url, request_type=GET_REQUEST_TYPE, request_body=''): 80 | try: 81 | url = '{0}://{1}:{2}{3}'.format(protocol, host, port, api_url) 82 | admin_auth = base64.encodestring('%s:%s' % (login, password)).replace('\n', '') 83 | request = urllib2.Request(url) 84 | request.add_header('Authorization', 'Basic %s' % admin_auth) 85 | request.add_header('X-Requested-By', 'ambari') 86 | request.add_data(request_body) 87 | request.get_method = lambda: request_type 88 | 89 | if unsafe: 90 | ctx = ssl.create_default_context() 91 | ctx.check_hostname = False 92 | ctx.verify_mode = ssl.CERT_NONE 93 | response = urllib2.urlopen(request, context=ctx) 94 | else: 95 | response = urllib2.urlopen(request) 96 | 97 | response_body = response.read() 98 | except Exception as exc: 99 | raise Exception('Problem with accessing api. Reason: {0}'.format(exc)) 100 | return response_body 101 | return do_request 102 | 103 | def get_config_tag(cluster, config_type, accessor): 104 | response = accessor(DESIRED_CONFIGS_URL.format(cluster)) 105 | try: 106 | desired_tags = json.loads(response) 107 | current_config_tag = desired_tags[CLUSTERS][DESIRED_CONFIGS][config_type][TAG] 108 | except Exception as exc: 109 | raise Exception('"{0}" not found in server response. Response:\n{1}'.format(config_type, response)) 110 | return current_config_tag 111 | 112 | def create_new_desired_config(cluster, config_type, properties, attributes, accessor, version_note): 113 | new_tag = TAG_PREFIX + str(int(time.time() * 1000000)) 114 | new_config = { 115 | CLUSTERS: { 116 | DESIRED_CONFIGS: { 117 | TYPE: config_type, 118 | TAG: new_tag, 119 | SERVICE_CONFIG_NOTE:version_note, 120 | PROPERTIES: properties 121 | } 122 | } 123 | } 124 | if len(attributes.keys()) > 0: 125 | new_config[CLUSTERS][DESIRED_CONFIGS][ATTRIBUTES] = attributes 126 | request_body = json.dumps(new_config) 127 | new_file = 'doSet_{0}.json'.format(new_tag) 128 | logger.info('### PUTting json into: {0}'.format(new_file)) 129 | output_to_file(new_file)(new_config) 130 | accessor(CLUSTERS_URL.format(cluster), PUT_REQUEST_TYPE, request_body) 131 | logger.info('### NEW Site:{0}, Tag:{1}'.format(config_type, new_tag)) 132 | 133 | def get_current_config(cluster, config_type, accessor): 134 | config_tag = get_config_tag(cluster, config_type, accessor) 135 | logger.info("### on (Site:{0}, Tag:{1})".format(config_type, config_tag)) 136 | response = accessor(CONFIGURATION_URL.format(cluster, config_type, config_tag)) 137 | config_by_tag = json.loads(response) 138 | current_config = config_by_tag[ITEMS][0] 139 | return current_config[PROPERTIES], current_config.get(ATTRIBUTES, {}) 140 | 141 | def update_config(cluster, config_type, config_updater, accessor, version_note): 142 | properties, attributes = config_updater(cluster, config_type, accessor) 143 | create_new_desired_config(cluster, config_type, properties, attributes, accessor, version_note) 144 | 145 | def update_specific_property(config_name, config_value): 146 | def update(cluster, config_type, accessor): 147 | properties, attributes = get_current_config(cluster, config_type, accessor) 148 | properties[config_name] = config_value 149 | return properties, attributes 150 | return update 151 | 152 | def update_from_xml(config_file): 153 | def update(cluster, config_type, accessor): 154 | return read_xml_data_to_map(config_file) 155 | return update 156 | 157 | # Used DOM parser to read data into a map 158 | def read_xml_data_to_map(path): 159 | configurations = {} 160 | properties_attributes = {} 161 | tree = ET.parse(path) 162 | root = tree.getroot() 163 | for properties in root.getiterator('property'): 164 | name = properties.find('name') 165 | value = properties.find('value') 166 | final = properties.find('final') 167 | 168 | if name != None: 169 | name_text = name.text if name.text else "" 170 | else: 171 | logger.warn("No name is found for one of the properties in {0}, ignoring it".format(path)) 172 | continue 173 | 174 | if value != None: 175 | value_text = value.text if value.text else "" 176 | else: 177 | logger.warn("No value is found for \"{0}\" in {1}, using empty string for it".format(name_text, path)) 178 | value_text = "" 179 | 180 | if final != None: 181 | final_text = final.text if final.text else "" 182 | properties_attributes[name_text] = final_text 183 | 184 | configurations[name_text] = value_text 185 | return configurations, {"final" : properties_attributes} 186 | 187 | def update_from_file(config_file): 188 | def update(cluster, config_type, accessor): 189 | try: 190 | with open(config_file) as in_file: 191 | file_content = in_file.read() 192 | except Exception as e: 193 | raise Exception('Cannot find file "{0}" to PUT'.format(config_file)) 194 | try: 195 | file_properties = json.loads(file_content) 196 | except Exception as e: 197 | raise Exception('File "{0}" should be in the following JSON format ("properties_attributes" is optional):\n{1}'.format(config_file, FILE_FORMAT)) 198 | new_properties = file_properties.get(PROPERTIES, {}) 199 | new_attributes = file_properties.get(ATTRIBUTES, {}) 200 | logger.info('### PUTting file: "{0}"'.format(config_file)) 201 | return new_properties, new_attributes 202 | return update 203 | 204 | def delete_specific_property(config_name): 205 | def update(cluster, config_type, accessor): 206 | properties, attributes = get_current_config(cluster, config_type, accessor) 207 | properties.pop(config_name, None) 208 | for attribute_values in attributes.values(): 209 | attribute_values.pop(config_name, None) 210 | return properties, attributes 211 | return update 212 | 213 | def output_to_file(filename): 214 | def output(config): 215 | with open(filename, 'w') as out_file: 216 | json.dump(config, out_file, indent=2) 217 | return output 218 | 219 | def output_to_console(config): 220 | print json.dumps(config, indent=2) 221 | 222 | def get_config(cluster, config_type, accessor, output): 223 | properties, attributes = get_current_config(cluster, config_type, accessor) 224 | config = {PROPERTIES: properties} 225 | if len(attributes.keys()) > 0: 226 | config[ATTRIBUTES] = attributes 227 | output(config) 228 | 229 | def set_properties(cluster, config_type, args, accessor, version_note): 230 | logger.info('### Performing "set":') 231 | 232 | if len(args) == 1: 233 | config_file = args[0] 234 | root, ext = os.path.splitext(config_file) 235 | if ext == ".xml": 236 | updater = update_from_xml(config_file) 237 | elif ext == ".json": 238 | updater = update_from_file(config_file) 239 | else: 240 | logger.error("File extension {0} is not supported".format(ext)) 241 | return -1 242 | logger.info('### from file {0}'.format(config_file)) 243 | else: 244 | config_name = args[0] 245 | config_value = args[1] 246 | updater = update_specific_property(config_name, config_value) 247 | logger.info('### new property - "{0}":"{1}"'.format(config_name, config_value)) 248 | update_config(cluster, config_type, updater, accessor, version_note) 249 | return 0 250 | 251 | def delete_properties(cluster, config_type, args, accessor, version_note): 252 | logger.info('### Performing "delete":') 253 | if len(args) == 0: 254 | logger.error("Not enough arguments. Expected config key.") 255 | return -1 256 | 257 | config_name = args[0] 258 | logger.info('### on property "{0}"'.format(config_name)) 259 | update_config(cluster, config_type, delete_specific_property(config_name), accessor, version_note) 260 | return 0 261 | 262 | 263 | def get_properties(cluster, config_type, args, accessor): 264 | logger.info("### Performing \"get\" content:") 265 | if len(args) > 0: 266 | filename = args[0] 267 | output = output_to_file(filename) 268 | logger.info('### to file "{0}"'.format(filename)) 269 | else: 270 | output = output_to_console 271 | get_config(cluster, config_type, accessor, output) 272 | return 0 273 | 274 | def main(): 275 | 276 | parser = optparse.OptionParser(usage="usage: %prog [options]") 277 | 278 | login_options_group = OptionGroup(parser, "To specify credentials please use \"-e\" OR \"-u\" and \"-p'\"") 279 | login_options_group.add_option("-u", "--user", dest="user", default="admin", help="Optional user ID to use for authentication. Default is 'admin'") 280 | login_options_group.add_option("-p", "--password", dest="password", default="admin", help="Optional password to use for authentication. Default is 'admin'") 281 | login_options_group.add_option("-e", "--credentials-file", dest="credentials_file", help="Optional file with user credentials separated by new line.") 282 | parser.add_option_group(login_options_group) 283 | 284 | parser.add_option("-t", "--port", dest="port", default="8080", help="Optional port number for Ambari server. Default is '8080'. Provide empty string to not use port.") 285 | parser.add_option("-s", "--protocol", dest="protocol", default="http", help="Optional support of SSL. Default protocol is 'http'") 286 | parser.add_option("--unsafe", action="store_true", dest="unsafe", help="Skip SSL certificate verification.") 287 | parser.add_option("-a", "--action", dest="action", help="Script action: , , ") 288 | parser.add_option("-l", "--host", dest="host", help="Server external host name") 289 | parser.add_option("-n", "--cluster", dest="cluster", help="Name given to cluster. Ex: 'c1'") 290 | parser.add_option("-c", "--config-type", dest="config_type", help="One of the various configuration types in Ambari. Ex: core-site, hdfs-site, mapred-queue-acls, etc.") 291 | parser.add_option("-b", "--version-note", dest="version_note", default="", help="Version change notes which will help to know what has been changed in this config. This value is optional and is used for actions and .") 292 | 293 | config_options_group = OptionGroup(parser, "To specify property(s) please use \"-f\" OR \"-k\" and \"-v'\"") 294 | config_options_group.add_option("-f", "--file", dest="file", help="File where entire configurations are saved to, or read from. Supported extensions (.xml, .json>)") 295 | config_options_group.add_option("-k", "--key", dest="key", help="Key that has to be set or deleted. Not necessary for 'get' action.") 296 | config_options_group.add_option("-v", "--value", dest="value", help="Optional value to be set. Not necessary for 'get' or 'delete' actions.") 297 | parser.add_option_group(config_options_group) 298 | 299 | (options, args) = parser.parse_args() 300 | 301 | logger.setLevel(logging.INFO) 302 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') 303 | stdout_handler = logging.StreamHandler(sys.stdout) 304 | stdout_handler.setLevel(logging.INFO) 305 | stdout_handler.setFormatter(formatter) 306 | logger.addHandler(stdout_handler) 307 | 308 | # options with default value 309 | 310 | if not options.credentials_file and (not options.user or not options.password): 311 | parser.error("You should use option (-e) to set file with Ambari user credentials OR use (-u) username and (-p) password") 312 | 313 | if options.credentials_file: 314 | if os.path.isfile(options.credentials_file): 315 | try: 316 | with open(options.credentials_file) as credentials_file: 317 | file_content = credentials_file.read() 318 | login_lines = filter(None, file_content.splitlines()) 319 | if len(login_lines) == 2: 320 | user = login_lines[0] 321 | password = login_lines[1] 322 | else: 323 | logger.error("Incorrect content of {0} file. File should contain Ambari username and password separated by new line.".format(options.credentials_file)) 324 | return -1 325 | except Exception as e: 326 | logger.error("You don't have permissions to {0} file".format(options.credentials_file)) 327 | return -1 328 | else: 329 | logger.error("File {0} doesn't exist or you don't have permissions.".format(options.credentials_file)) 330 | return -1 331 | else: 332 | user = options.user 333 | password = options.password 334 | 335 | port = options.port 336 | protocol = options.protocol 337 | 338 | #options without default value 339 | if None in [options.action, options.host, options.cluster, options.config_type]: 340 | parser.error("One of required options is not passed") 341 | 342 | action = options.action 343 | host = options.host 344 | cluster = options.cluster 345 | config_type = options.config_type 346 | version_note = options.version_note 347 | 348 | accessor = api_accessor(host, user, password, protocol, port, options.unsafe) 349 | if action == SET_ACTION: 350 | 351 | if not options.file and (not options.key or not options.value): 352 | parser.error("You should use option (-f) to set file where entire configurations are saved OR (-k) key and (-v) value for one property") 353 | if options.file: 354 | action_args = [options.file] 355 | else: 356 | action_args = [options.key, options.value] 357 | return set_properties(cluster, config_type, action_args, accessor, version_note) 358 | 359 | elif action == GET_ACTION: 360 | if options.file: 361 | action_args = [options.file] 362 | else: 363 | action_args = [] 364 | return get_properties(cluster, config_type, action_args, accessor) 365 | 366 | elif action == DELETE_ACTION: 367 | if not options.key: 368 | parser.error("You should use option (-k) to set property name witch will be deleted") 369 | else: 370 | action_args = [options.key] 371 | return delete_properties(cluster, config_type, action_args, accessor, version_note) 372 | else: 373 | logger.error('Action "{0}" is not supported. Supported actions: "get", "set", "delete".'.format(action)) 374 | return -1 375 | 376 | if __name__ == "__main__": 377 | try: 378 | sys.exit(main()) 379 | except (KeyboardInterrupt, EOFError): 380 | print("\nAborting ... Keyboard Interrupt.") 381 | sys.exit(1) 382 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | def left(field, length): 2 | diff = length - len(str(field)) 3 | return str(field) + " " * diff 4 | 5 | 6 | def center(field, length): 7 | if isinstance(field, list): 8 | diff = length - len(str(field[0])) 9 | return " " * (diff / 2) + str(field[0]) + " " * (length - len(str(field[0])) - (diff / 2)) 10 | else: 11 | diff = length - len(str(field)) 12 | return " " * (diff / 2) + str(field) + " " * (length - len(str(field)) - (diff / 2)) 13 | 14 | 15 | def right(field, length): 16 | diff = length - len(str(field)) 17 | return " " * diff + str(field) 18 | 19 | 20 | def pprinttable(rows, fields): 21 | output = buildtable(rows, fields) 22 | for line in output: 23 | print line 24 | return output 25 | 26 | def buildtable(rows, fields): 27 | str_list = [] 28 | 29 | if len(rows) > 0: 30 | # headers = HEADER._fields 31 | # headers = HEADER 32 | lens = [] 33 | for field in fields: 34 | lens.append(len(field[1])) 35 | 36 | for row in rows: 37 | inc = 0 38 | for field in fields: 39 | if isinstance(row[field[0]], (int, float, long)): 40 | if lens[inc] < 4: 41 | lens[inc] = 4 42 | if lens[inc] < len(str(row[field[0]])): 43 | lens[inc] = len(str(row[field[0]])) 44 | # if lens[inc] < 16: 45 | # lens[inc] = 16 46 | elif isinstance(row[field[0]], (list, tuple)): 47 | size = 2 48 | for i in range(len(row[field[0]])): 49 | size += len(row[field[0]][i]) + 3 50 | if size > lens[inc]: 51 | lens[inc] = size 52 | elif isinstance(row[field[0]], (dict)): 53 | size = 2 54 | for i in range(len(row[field[0]])): 55 | size += len(row[field[0]]) + 3 56 | if size > lens[inc]: 57 | lens[inc] = size 58 | else: 59 | if row[field[0]] is not None and (len(row[field[0]]) > lens[inc]): 60 | lens[inc] = len(row[field[0]]) 61 | inc += 1 62 | 63 | headerRowSeparator = "" 64 | headerRow = "" 65 | for loc in range(len(fields)): 66 | headerRowSeparator = headerRowSeparator + "|" + "=" * (lens[loc]+1) 67 | headerRow = headerRow + "| " + center([fields[loc][1]], lens[loc]) 68 | 69 | headerRowSeparator = headerRowSeparator + "|" 70 | headerRow = headerRow + "|" 71 | 72 | str_list.append(headerRowSeparator) 73 | # print headerRowSeparator 74 | str_list.append(headerRow) 75 | # print headerRow 76 | str_list.append(headerRowSeparator) 77 | # print headerRowSeparator 78 | 79 | for row in rows: 80 | inc = 0 81 | recordRow = "" 82 | offset = 0 83 | for field in fields: 84 | if isinstance(row[field[0]], int) or isinstance(row[field[0]], float) or isinstance(row[field[0]], long): 85 | recordRow = recordRow + "| " + right(row[field[0]], lens[inc]) 86 | # elif isinstance(row[field[0]], bool): 87 | # if row[field[0]]: 88 | # recordRow = recordRow + "| " + right('X', lens[inc]) 89 | # else: 90 | # recordRow = recordRow + "| " + right('', lens[inc]) 91 | 92 | elif isinstance(row[field[0]], (dict)): 93 | # recordRow = recordRow + "| " 94 | offset = len(recordRow) 95 | it = 0 96 | for item in row[field[0]]: 97 | dictItem = str(row[field[0]][item]) 98 | if it == 0: 99 | recordRow = recordRow + '|' + left(dictItem, lens[inc] + 1) + '|\n|' 100 | elif it == len(row[field[0]]) - 1: 101 | recordRow = recordRow + ' '.rjust(offset-1) + '|' + left(dictItem, lens[inc] + 1) 102 | else: 103 | recordRow = recordRow + ' '.rjust(offset-1) + '|' + left(dictItem, lens[inc] + 1) + '|\n|' 104 | it += 1 105 | else: 106 | recordRow = recordRow + "| " + left(row[field[0]], lens[inc]) 107 | inc += 1 108 | recordRow = recordRow + "|" 109 | 110 | str_list.append(recordRow) 111 | # print recordRow 112 | 113 | str_list.append(headerRowSeparator) 114 | # print headerRowSeparator 115 | return str_list 116 | 117 | 118 | def pprinttable2(rows, fields): 119 | output = buildtable2(rows, fields) 120 | for line in output: 121 | print line 122 | 123 | 124 | def buildtable2(rows, fields): 125 | str_list = [] 126 | 127 | if len(rows) > 0: 128 | # headers = HEADER._fields 129 | # headers = HEADER 130 | lens = [] 131 | for field in fields: 132 | lens.append(len(field)) 133 | 134 | for row in rows: 135 | inc = 0 136 | for field in fields: 137 | try: 138 | value = row[field] 139 | if isinstance(row[field], (int, float, long)): 140 | if lens[inc] < 4: 141 | lens[inc] = 4 142 | if lens[inc] < len(str(row[field])): 143 | lens[inc] = len(str(row[field])) 144 | # if lens[inc] < 16: 145 | # lens[inc] = 16 146 | elif isinstance(row[field], (list, tuple)): 147 | size = 2 148 | for i in range(len(row[field])): 149 | size += len(row[field][i]) + 3 150 | if size > lens[inc]: 151 | lens[inc] = size 152 | elif isinstance(row[field], (dict)): 153 | size = 2 154 | for i in range(len(row[field])): 155 | size += len(row[field]) + 3 156 | if size > lens[inc]: 157 | lens[inc] = size 158 | else: 159 | if row[field] is not None and (len(row[field]) > lens[inc]): 160 | lens[inc] = len(row[field]) 161 | except: 162 | pass 163 | inc += 1 164 | 165 | headerRowSeparator = "" 166 | headerRow = "" 167 | loc = 0 168 | for field in fields: 169 | # for loc in range(len(fields)): 170 | headerRowSeparator = headerRowSeparator + "|" + "=" * (lens[loc]+1) 171 | headerRow = headerRow + "| " + center(field, lens[loc]) 172 | loc += 1 173 | 174 | headerRowSeparator = headerRowSeparator + "|" 175 | headerRow = headerRow + "|" 176 | 177 | str_list.append(headerRowSeparator) 178 | # print headerRowSeparator 179 | str_list.append(headerRow) 180 | # print headerRow 181 | str_list.append(headerRowSeparator) 182 | # print headerRowSeparator 183 | 184 | for row in rows: 185 | inc = 0 186 | recordRow = "" 187 | offset = 0 188 | for field in fields: 189 | try: 190 | value = row[field] 191 | if isinstance(row[field], int) or isinstance(row[field], float) or isinstance(row[field], long): 192 | recordRow = recordRow + "| " + right(row[field], lens[inc]) 193 | # elif isinstance(row[field[0]], bool): 194 | # if row[field[0]]: 195 | # recordRow = recordRow + "| " + right('X', lens[inc]) 196 | # else: 197 | # recordRow = recordRow + "| " + right('', lens[inc]) 198 | 199 | elif isinstance(row[field], (dict)): 200 | # recordRow = recordRow + "| " 201 | offset = len(recordRow) 202 | it = 0 203 | for item in row[field]: 204 | dictItem = str(item) + ':' + str(row[field][item]) 205 | if it == 0: 206 | recordRow = recordRow + '|' + left(dictItem, lens[inc] + 1) + '|\n|' 207 | elif it == len(row[field]) - 1: 208 | recordRow = recordRow + ' '.rjust(offset-1) + '|' + left(dictItem, lens[inc] + 1) 209 | else: 210 | recordRow = recordRow + ' '.rjust(offset-1) + '|' + left(dictItem, lens[inc] + 1) + '|\n|' 211 | it += 1 212 | else: 213 | recordRow = recordRow + "| " + left(row[field], lens[inc]) 214 | except: 215 | recordRow = recordRow + "| " + left(' ', lens[inc]) 216 | 217 | inc += 1 218 | 219 | recordRow = recordRow + "|" 220 | 221 | str_list.append(recordRow) 222 | # print recordRow 223 | 224 | str_list.append(headerRowSeparator) 225 | # print headerRowSeparator 226 | return str_list 227 | -------------------------------------------------------------------------------- /config_llap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import optparse 4 | from optparse import OptionGroup 5 | import logging 6 | import sys 7 | import os 8 | import json 9 | from ambari_configs import api_accessor, get_properties 10 | from common import pprinttable, buildtable 11 | from cStringIO import StringIO 12 | import math 13 | import datetime 14 | 15 | # Version used to display app version. 16 | # Using Hive Version as the base and "_" as the revision. 17 | VERSION = "3.1_11" 18 | 19 | logger = logging.getLogger('LLAPConfig') 20 | 21 | TF = ("true", "false") 22 | KB = 1024 23 | GB = 1024^3 24 | 25 | # SSL_ON = False 26 | SSL_CMD = "" 27 | 28 | TYPE_INPUT = "simple" 29 | TYPE_CALC = "calc" 30 | TYPE_REFERENCE = "extended" 31 | # TYPE_THRESHOLD = "threshold" 32 | 33 | # Yarn Sizing 34 | YARN_MEMORY_PERCENT = 80 35 | LLAP_CACHE_PERCENTAGE = 50 36 | 37 | # Configuration 38 | 39 | # [ Short Desc, Type, Section, Config, Value, Current Value, Options, Long Desc ] 40 | HEADER = ["Short Desc", "Type", "Section", "Config", "Value", "Current Value", "Options", "Long Desc"] 41 | 42 | # Positions 43 | # LOCATION,DISPLAY_ORDER 44 | POS_SHORT_DESC = [0,"Short Desc"] 45 | POS_TYPE = [1,"Type"] 46 | POS_SECTION = [2,"Section"] 47 | POS_CONFIG = [3,"Config"] 48 | POS_VALUE = [4,"Value"] 49 | POS_CUR_VALUE = [5,"Current Value"] 50 | POS_OPTIONS = [6,"Options"] 51 | POS_LONG_DESC = [7,"Long Desc"] 52 | POS_DELTA = [8, "Delta"] 53 | 54 | ALL_DISPLAY_COLUMNS = [POS_SHORT_DESC,POS_TYPE,POS_SECTION,POS_CONFIG, 55 | POS_VALUE,POS_CUR_VALUE,POS_OPTIONS,POS_LONG_DESC,POS_DELTA] 56 | 57 | DISPLAY_COLUMNS = [POS_SHORT_DESC, POS_SECTION, POS_CONFIG, 58 | POS_VALUE] 59 | 60 | # Sections 61 | YARN_SITE = ("YARN Configuration", "yarn-site") 62 | HIVE_INTERACTIVE_SITE = ("LLAP Configuration", "hive-interactive-site") 63 | HIVE_INTERACTIVE_ENV = ("LLAP Environment", "hive-interactive-env") 64 | TEZ_INTERACTIVE_SITE = ("LLAP Tez Configuration", "tez-interactive-site") 65 | HOST_ENV = ("Cluster Host Configuration", "host-env") 66 | THRESHOLD_ENV = ("Calculation Thresholds", "threshold-env") 67 | CLUSTER_ENV = ("Cluster Environment", "") 68 | 69 | VALID_AMBARI_SECTIONS = (YARN_SITE, HIVE_INTERACTIVE_SITE, HIVE_INTERACTIVE_ENV, TEZ_INTERACTIVE_SITE) 70 | 71 | SECTIONS = (HOST_ENV, THRESHOLD_ENV, YARN_SITE, HIVE_INTERACTIVE_SITE, HIVE_INTERACTIVE_ENV, TEZ_INTERACTIVE_SITE) 72 | 73 | # Environment 74 | WORKER_MEMORY_GB = ["Node Memory Footprint(GB)", TYPE_INPUT, HOST_ENV, 75 | "", 0, 0, (), "", 0] 76 | WORKER_COUNT = ["Number of Cluster Worker Nodes", TYPE_INPUT, HOST_ENV, 77 | "", 0, 0, (), "", 0] 78 | WORKER_CORES = ["YARN Resource CPU-vCores", TYPE_INPUT, YARN_SITE, "yarn.nodemanager.resource.cpu-vcores", 0, 0, (), "", 0] 79 | 80 | # Thresholds 81 | PERCENT_OF_HOST_MEM_FOR_YARN = ["Percent of Host Memory for YARN NodeManager", TYPE_REFERENCE, THRESHOLD_ENV, "", 80, None, (), "","na"] 82 | # PERCENT_OF_CLUSTER_FOR_LLAP = ["Percent of Cluster for LLAP", TYPE_CALC, THRESHOLD_ENV, "", 50, None] 83 | # PERCENT_OF_NODE_FOR_LLAP_MEM = ["Percent of NodeManager Memory for LLAP", TYPE_REFERENCE, THRESHOLD_ENV, "", 90, None] 84 | PERCENT_OF_LLAP_FOR_CACHE = ["Percent of LLAP Memory for Cache", TYPE_REFERENCE, THRESHOLD_ENV, "", 50, None, (), "","na"] 85 | PERCENT_OF_CORES_FOR_EXECUTORS = ["Percent of 'YARN vCores' for LLAP Executors", TYPE_REFERENCE, THRESHOLD_ENV, "", 100, None, (), "Default assumes dedicated use of compute node","na"] 86 | MAX_HEADROOM_GB = ["MAX LLAP Headroom Value(GB)", TYPE_REFERENCE, THRESHOLD_ENV, "", 12, None, (), "","na"] 87 | 88 | LLAP_DAEMON_CONTAINER_SAFETY_GB = ["Max LLAP YARN Container Size (GB) before applying 'Safety Valve'", TYPE_REFERENCE, THRESHOLD_ENV, "", 256, None, (), "","na"] 89 | LLAP_SAFETY_VALVE_MB = ["Unallocated YARN Container Memory (MB) for 'Safety Value'", TYPE_REFERENCE, THRESHOLD_ENV, "", 6192, None, (), "","na"] 90 | LLAP_TASK_MB_PER_INSTANCE_REFERENCE = ["Reference LLAP Task MB Allocation", TYPE_REFERENCE, THRESHOLD_ENV, "", 4096, None, (), "The reference memory amount per Executor Task. (in megabytes).", 0] 91 | 92 | PERCENT_OF_DAEMON_CONTAINER_MEM_MB_FOR_HEADROOM = ["Percent of Daemon Container Memory(MB) for Headroom", 93 | TYPE_REFERENCE, THRESHOLD_ENV, "", 20, None, (), "","na"] 94 | PERCENT_OF_EXECUTORS_FOR_IO_THREADPOOL = ["Percent of Executors for IO Threadpool", TYPE_REFERENCE, 95 | THRESHOLD_ENV, "", 100, None, (), "","na"] 96 | 97 | # YARN 98 | YARN_NM_RSRC_MEM_MB = ["Node Manager Memory(MB)", TYPE_CALC, YARN_SITE, 99 | "yarn.nodemanager.resource.memory-mb", 0, 0, (), "", 0] 100 | YARN_SCH_MAX_ALLOC_MEM_MB = ["Yarn Max Mem Allocation(MB)", TYPE_CALC, YARN_SITE, 101 | "yarn.scheduler.maximum-allocation-mb", 0, 0, (), "", 0] 102 | YARN_SCH_MIN_ALLOC_MEM_MB = ["Yarn Min Mem Allocation(MB)", TYPE_REFERENCE, YARN_SITE, 103 | "yarn.scheduler.minimum-allocation-mb", 1024, 1024, (), "", 0] 104 | 105 | DIVIDER = ["", "", "", "", "", "", "", "", ""] 106 | THRESHOLDS = [" -- Threshold DETAILS -- ", "", "", "", "", "", "", "", ""] 107 | CLUSTER_DETAILS = [" -- CLUSTER DETAILS -- ", "", "", "", "", "", "", "", ""] 108 | YARN_ENV = [" -- YARN DETAILS --", "", "", "", "", "", "", "", ""] 109 | HIVE_ENV = [" -- HIVE DETAILS --", "", "", "", "", "", "", "", ""] 110 | TOTALS = ["> Totals", "", "", "", "", "", "", "", ""] 111 | 112 | 113 | # Hive Interactive Site 114 | HIVE_LLAP_QUEUE = ["YARN Queue", TYPE_INPUT, HIVE_INTERACTIVE_SITE, 115 | "hive.llap.daemon.queue.name", "llap", "llap", (), "","na"] 116 | # wip 117 | # HIVE_LLAP_CONCURRENCY_QUEUE = ["YARN Queue", TYPE_INPUT, HIVE_INTERACTIVE_SITE, 118 | # "hive.server2.tez.default.queues", "llap", "llap", (), "","na"] 119 | # HIVE_LLAP_CONCURRENCY_QUEUE_ALLOW_CUSTOM = ["Allow Custom Concurrency AM Queue", TYPE_INPUT, HIVE_INTERACTIVE_SITE, 120 | # "hive.server2.tez.sessions.custom.queue.allowed", "false", "false", (), "", "na"] 121 | # HIVE_LLAP_WORKLOAD_MANAGER_QUEUE = ["LLAP Workload Management Queue", TYPE_INPUT, HIVE_INTERACTIVE_SITE, 122 | # "hive.server2.tez.interactive.queue", "", "", (), "When defined, concurrency determined by Workload Management Parallelism", "na"] 123 | 124 | TEZ_CONTAINER_SIZE_MB = ["TEZ Container Size", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 125 | "hive.tez.container.size", KB * 4, KB * 4, (), "Tez container size when LLAP is run with hive.execution.mode=container, which launches container instances for the job.",0] 126 | 127 | LLAP_DAEMON_CONTAINER_MEM_MB = ["Daemon Memory(MB)", TYPE_CALC, HIVE_INTERACTIVE_SITE, 128 | "hive.llap.daemon.yarn.container.mb", 0, 0, (), "", 0] 129 | 130 | LLAP_CACHE_MEM_MB = ["Cache(MB)", TYPE_CALC, HIVE_INTERACTIVE_SITE, 131 | "hive.llap.io.memory.size", 0, 0, (), "(LLAP Cache) Maximum size for IO allocator or ORC low-level cache.", 0] 132 | LLAP_OBJECT_CACHE_ENABLED = ["Object Cache Enabled?", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 133 | "hive.llap.object.cache.enabled", "true", "true", 134 | TF, "Cache objects (plans, hashtables, etc) in LLAP","na"] 135 | LLAP_MEMORY_MODE = ["Memory Mode", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 136 | "hive.llap.io.memory.mode", "cache", "cache", ("cache", "allocator", "none"), "","na"] 137 | LLAP_IO_ENABLED = ["Cache Enabled?", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 138 | "hive.llap.io.enabled", "true", "true", TF, "","na"] 139 | LLAP_IO_ALLOCATOR_NMAP_ENABLED = ["Direct I/O cache enabled?", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 140 | "hive.llap.io.allocator.mmap", "false", "false", TF, 141 | "Whether ORC low-level cache should use memory mapped allocation (direct I/O)","na"] 142 | LLAP_IO_ALLOCATOR_NMAP_PATH = ["Direct I/O cache path", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 143 | "hive.llap.io.allocator.mmap.path", "", "", (), "","na"] 144 | 145 | LLAP_NUM_EXECUTORS_PER_DAEMON = ["Num of Executors", TYPE_CALC, HIVE_INTERACTIVE_SITE, 146 | "hive.llap.daemon.num.executors", 0, 0, (), "",0] 147 | 148 | LLAP_IO_THREADPOOL = ["I/O Threadpool", TYPE_CALC, HIVE_INTERACTIVE_SITE, 149 | "hive.llap.io.threadpool.size", 0, 0, (), "", 0] 150 | 151 | 152 | # Hive Interactive Size (Custom) 153 | LLAP_PREWARMED_ENABLED = ["Prewarmed Containers", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 154 | "hive.prewarm.enabled", "false", "false", TF, "", "na"] 155 | LLAP_PREWARM_NUM_CONTAINERS = ["Number of prewarmed Containers", TYPE_REFERENCE, HIVE_INTERACTIVE_SITE, 156 | "hive.prewarm.numcontainers", 0, 0, (), "", 1] 157 | 158 | # Hive Interactive Env 159 | LLAP_NUM_NODES = ["Daemon Count", TYPE_INPUT, HIVE_INTERACTIVE_ENV, 160 | "num_llap_nodes", 0, 0, (), "", 0] 161 | LLAP_NUM_NODES_ALT = ["Daemon Count(legacy)", TYPE_CALC, HIVE_INTERACTIVE_ENV, 162 | "num_llap_nodes_for_llap_daemons", 0, 0, (), "", 0] 163 | LLAP_CONCURRENCY = ["Query Concurrency", TYPE_INPUT, HIVE_INTERACTIVE_SITE, 164 | "hive.server2.tez.sessions.per.default.queue", 0, 0, (), "", 0] 165 | LLAP_AM_DAEMON_HEAP_MB = ["AM Heap for Daemons", TYPE_REFERENCE, HIVE_INTERACTIVE_ENV, 166 | "hive_heapsize", 4096, 0, (), "Could be 2048, but defaults to 4096 in Ambari", 0] 167 | LLAP_HEADROOM_MEM_MB = ["Heap Headroom", TYPE_CALC, HIVE_INTERACTIVE_ENV, 168 | "llap_headroom_space", 0, 0, (), "", 0] 169 | LLAP_DAEMON_HEAP_MEM_MB = ["Daemon Heap size(MB)", TYPE_CALC, HIVE_INTERACTIVE_ENV, 170 | "llap_heap_size", 0, 0, (), "", 0] 171 | 172 | # TEZ Interactive site 173 | TEZ_AM_MEM_MB = ["TEZ AM Container size(MB) DAG Submission", TYPE_REFERENCE, TEZ_INTERACTIVE_SITE, 174 | "tez.am.resource.memory.mb", 4 * KB, 4 * KB, (), "", 0] 175 | 176 | # Total Section 177 | TOTAL_MEM_FOOTPRINT = ["Total Memory Footprint", TYPE_CALC, CLUSTER_ENV, 178 | "", 0, 0, (), "", 0] 179 | TOTAL_LLAP_DAEMON_FOOTPRINT = ["Total LLAP Daemon Memory Footprint", TYPE_CALC, CLUSTER_ENV, 180 | "", 0, 0, (), "", 0] 181 | TOTAL_LLAP_OTHER_FOOTPRINT = ["Total LLAP Other Memory Footprint", TYPE_CALC, CLUSTER_ENV, 182 | "", 0, 0, (), "", 0] 183 | TOTAL_LLAP_MEM_FOOTPRINT = ["Total LLAP Memory Footprint", TYPE_CALC, CLUSTER_ENV, 184 | "", 0, 0, (), "", 0] 185 | LLAP_QUEUE_MIN_REQUIREMENT = ["LLAP Minimum YARN Queue Capacity % Requirement", TYPE_CALC, CLUSTER_ENV, 186 | "", 0, 0, (), "", 0] 187 | 188 | LOGICAL_CONFIGS = [ 189 | CLUSTER_DETAILS, 190 | WORKER_MEMORY_GB, 191 | WORKER_COUNT, 192 | WORKER_CORES, 193 | DIVIDER, 194 | THRESHOLDS, 195 | PERCENT_OF_HOST_MEM_FOR_YARN, 196 | # PERCENT_OF_CLUSTER_FOR_LLAP, 197 | # PERCENT_OF_NODE_FOR_LLAP_MEM, 198 | LLAP_DAEMON_CONTAINER_SAFETY_GB, 199 | LLAP_SAFETY_VALVE_MB, 200 | LLAP_TASK_MB_PER_INSTANCE_REFERENCE, 201 | PERCENT_OF_LLAP_FOR_CACHE, 202 | PERCENT_OF_CORES_FOR_EXECUTORS, 203 | PERCENT_OF_DAEMON_CONTAINER_MEM_MB_FOR_HEADROOM, 204 | PERCENT_OF_EXECUTORS_FOR_IO_THREADPOOL, 205 | MAX_HEADROOM_GB, 206 | DIVIDER, 207 | YARN_ENV, 208 | YARN_NM_RSRC_MEM_MB, 209 | YARN_SCH_MAX_ALLOC_MEM_MB, 210 | YARN_SCH_MIN_ALLOC_MEM_MB, 211 | HIVE_LLAP_QUEUE, 212 | TOTALS, 213 | TOTAL_MEM_FOOTPRINT, 214 | DIVIDER, 215 | HIVE_ENV, 216 | LLAP_NUM_NODES, 217 | LLAP_NUM_NODES_ALT, 218 | LLAP_CONCURRENCY, 219 | TEZ_AM_MEM_MB, 220 | LLAP_NUM_EXECUTORS_PER_DAEMON, 221 | DIVIDER, 222 | LLAP_AM_DAEMON_HEAP_MB, 223 | LLAP_DAEMON_CONTAINER_MEM_MB, 224 | LLAP_DAEMON_HEAP_MEM_MB, 225 | LLAP_HEADROOM_MEM_MB, 226 | LLAP_CACHE_MEM_MB, 227 | DIVIDER, 228 | LLAP_MEMORY_MODE, 229 | LLAP_IO_ENABLED, 230 | LLAP_OBJECT_CACHE_ENABLED, 231 | DIVIDER, 232 | 233 | LLAP_IO_THREADPOOL, 234 | DIVIDER, 235 | 236 | LLAP_IO_ALLOCATOR_NMAP_ENABLED, 237 | LLAP_IO_ALLOCATOR_NMAP_PATH, 238 | TEZ_CONTAINER_SIZE_MB, 239 | DIVIDER, 240 | 241 | LLAP_PREWARMED_ENABLED, 242 | LLAP_PREWARM_NUM_CONTAINERS, 243 | DIVIDER, 244 | 245 | TOTALS, 246 | TOTAL_LLAP_DAEMON_FOOTPRINT, 247 | TOTAL_LLAP_OTHER_FOOTPRINT, 248 | TOTAL_LLAP_MEM_FOOTPRINT, 249 | 250 | 251 | DIVIDER, 252 | LLAP_QUEUE_MIN_REQUIREMENT] 253 | 254 | AMBARI_CONFIGS = [ 255 | YARN_NM_RSRC_MEM_MB, 256 | 257 | YARN_SCH_MAX_ALLOC_MEM_MB, 258 | YARN_SCH_MIN_ALLOC_MEM_MB, 259 | WORKER_CORES, 260 | HIVE_LLAP_QUEUE, 261 | LLAP_QUEUE_MIN_REQUIREMENT, 262 | 263 | LLAP_NUM_NODES, 264 | LLAP_NUM_NODES_ALT, 265 | 266 | LLAP_CONCURRENCY, 267 | TEZ_AM_MEM_MB, 268 | 269 | LLAP_NUM_EXECUTORS_PER_DAEMON, 270 | 271 | LLAP_AM_DAEMON_HEAP_MB, 272 | LLAP_DAEMON_CONTAINER_MEM_MB, 273 | LLAP_DAEMON_HEAP_MEM_MB, 274 | LLAP_HEADROOM_MEM_MB, 275 | LLAP_CACHE_MEM_MB, 276 | 277 | LLAP_MEMORY_MODE, 278 | LLAP_IO_ENABLED, 279 | LLAP_OBJECT_CACHE_ENABLED, 280 | 281 | LLAP_IO_THREADPOOL, 282 | 283 | LLAP_IO_ALLOCATOR_NMAP_ENABLED, 284 | LLAP_IO_ALLOCATOR_NMAP_PATH, 285 | 286 | TEZ_CONTAINER_SIZE_MB, 287 | 288 | LLAP_PREWARMED_ENABLED, 289 | LLAP_PREWARM_NUM_CONTAINERS 290 | ] 291 | 292 | SELECT_TASK = "Select Task -- : " 293 | SELECT_SECTION = "Select Section -- : " 294 | SELECT_MODE = "Select Mode -- : " 295 | SELECT_ACTION = "Select Action --: " 296 | SELECT_CONFIG = "Select Config -- : " 297 | ENTER_RETURN = " - to go back" 298 | ENTER_CONTINUE = " - to continue" 299 | AMBARI_CFG_CMD = "./ambari_configs.py --host=${{AMBARI_HOST}} --port=${{AMBARI_PORT}} {0} --cluster=${{CLUSTER_NAME}}" + \ 300 | " --credentials-file=${{HOME}}/.ambari-credentials --action=set --config-type={1}" + \ 301 | " --key={2} --value={3}" 302 | AMBARI_CFG_CMD_V = "./ambari_configs.py --host=${{AMBARI_HOST}} --port=${{AMBARI_PORT}} {0} --cluster=${{CLUSTER_NAME}}" + \ 303 | " --credentials-file=${{HOME}}/.ambari-credentials --action=set --config-type={1}" + \ 304 | " --key={2} --value={3} --version-note \"{4}\"" 305 | 306 | 307 | ISSUE_MESSAGES = [] 308 | RECOMMENDATION_TYPE = "RECOMMENDATION" 309 | WARNING_TYPE = "WARNING" 310 | RULE_APPLICATION_TYPE = "RULE APPLIED" 311 | ERROR_TYPE = "ERROR" 312 | 313 | cluster = "" 314 | ambari_accessor_api = None 315 | version_note = "" 316 | ambari_integration = True 317 | current_values = True 318 | 319 | MODE = [TYPE_INPUT] 320 | 321 | 322 | class Capturing(list): 323 | def __enter__(self): 324 | self._stdout = sys.stdout 325 | sys.stdout = self._stringio = StringIO() 326 | return self 327 | 328 | def __exit__(self, *args): 329 | self.extend(self._stringio.getvalue().splitlines()) 330 | del self._stringio # free up some memory 331 | sys.stdout = self._stdout 332 | 333 | 334 | def calc_prerequisite(): 335 | if WORKER_CORES[POS_VALUE[0]] < 1: 336 | return False 337 | if WORKER_COUNT[POS_VALUE[0]] < 1: 338 | return False 339 | if WORKER_MEMORY_GB[POS_VALUE[0]] < 1: 340 | return False 341 | if LLAP_NUM_NODES[POS_VALUE[0]] < 1: 342 | return False 343 | if LLAP_CONCURRENCY[POS_VALUE[0]] < 1: 344 | return False 345 | 346 | return True 347 | 348 | 349 | def run_calc(position): 350 | # print ("running calc") 351 | if not calc_prerequisite(): 352 | check_for_issues() 353 | return 354 | 355 | ## YARN 356 | ######### 357 | # YARN_NM_RSRC_MEM_MB 358 | YARN_NM_RSRC_MEM_MB[position] = WORKER_MEMORY_GB[position] * KB * PERCENT_OF_HOST_MEM_FOR_YARN[position] / 100 359 | 360 | ## LLAP 361 | ######### 362 | # LLAP_NUM_NODES 363 | # LLAP_NUM_NODES[position] = WORKER_COUNT[position] * PERCENT_OF_CLUSTER_FOR_LLAP[position] / 100 364 | # Sync Values. 365 | LLAP_NUM_NODES_ALT[position] = LLAP_NUM_NODES[position] 366 | 367 | # LLAP_NUM_EXECUTORS_PER_DAEMON 368 | LLAP_NUM_EXECUTORS_PER_DAEMON[position] = WORKER_CORES[position] * \ 369 | PERCENT_OF_CORES_FOR_EXECUTORS[position] / 100 370 | 371 | # Total LLAP Other Footprint 372 | # TODO: Add support for PREWARMED CONTAINERS. 373 | TOTAL_LLAP_OTHER_FOOTPRINT[position] = LLAP_CONCURRENCY[position] * TEZ_AM_MEM_MB[position] 374 | 375 | OTHER_MEM_PER_NODE = TOTAL_LLAP_OTHER_FOOTPRINT[position] / LLAP_NUM_NODES[position] 376 | 377 | # LLAP_DAEMON_CONTAINER_MEM_MB 378 | LLAP_DAEMON_CONTAINER_MEM_MB[position] = YARN_NM_RSRC_MEM_MB[position] - OTHER_MEM_PER_NODE 379 | 380 | # YARN_SCH_MAX_ALLOC_MEM_MB 381 | # Adding a percent for a little clearance on mem settings. 382 | YARN_SCH_MAX_ALLOC_MEM_MB[position] = LLAP_DAEMON_CONTAINER_MEM_MB[position] + 1 383 | 384 | # LLAP_HEADROOM_MEM_MB 385 | # =IF((E37*0.2) > (12*1024),12*1024,E37*0.2) 386 | if LLAP_DAEMON_CONTAINER_MEM_MB[position] * PERCENT_OF_DAEMON_CONTAINER_MEM_MB_FOR_HEADROOM[position] / 100 \ 387 | > MAX_HEADROOM_GB[position] * KB: 388 | LLAP_HEADROOM_MEM_MB[position] = MAX_HEADROOM_GB[position] * KB 389 | else: 390 | LLAP_HEADROOM_MEM_MB[position] = LLAP_DAEMON_CONTAINER_MEM_MB[position] * \ 391 | PERCENT_OF_DAEMON_CONTAINER_MEM_MB_FOR_HEADROOM[position] / 100 392 | 393 | # LLAP_DAEMON_HEAP_MEM_MB 394 | LLAP_DAEMON_HEAP_MEM_MB[position] = (LLAP_DAEMON_CONTAINER_MEM_MB[position] - LLAP_HEADROOM_MEM_MB[position]) * \ 395 | (100 - PERCENT_OF_LLAP_FOR_CACHE[position]) / 100 396 | 397 | # This ensures that a minimum of 4Gb per Executor is available in the Daemon Heap. 398 | if LLAP_DAEMON_HEAP_MEM_MB[position] < LLAP_NUM_EXECUTORS_PER_DAEMON[position] * LLAP_TASK_MB_PER_INSTANCE_REFERENCE[position]: 399 | LLAP_DAEMON_HEAP_MEM_MB[position] = LLAP_NUM_EXECUTORS_PER_DAEMON[position] * LLAP_TASK_MB_PER_INSTANCE_REFERENCE[position] 400 | 401 | 402 | # LLAP_CACHE_MEM_MB 403 | LLAP_CACHE_MEM_MB[position] = LLAP_DAEMON_CONTAINER_MEM_MB[position] - LLAP_DAEMON_HEAP_MEM_MB[position] 404 | 405 | # If we've exceeded the Container Threshold, add in a buffer to avoid YARN Killing Containers under load. 406 | # LLAP_DAEMON_CONTAINER_SAFETY_GB = ["Max LLAP YARN Container Size before applying 'Safety Valve'", TYPE_REFERENCE, THRESHOLD_ENV, "", 256, None, (), "","na"] 407 | # LLAP_SAFETY_VALVE_MB = ["Unallocated YARN Container Memory for grace", TYPE_REFERENCE, THRESHOLD_ENV, "", 6192, None, (), "","na"] 408 | if LLAP_DAEMON_CONTAINER_MEM_MB[position] >= LLAP_DAEMON_CONTAINER_SAFETY_GB[position] * KB: 409 | if LLAP_CACHE_MEM_MB[position] > LLAP_SAFETY_VALVE_MB[position]: 410 | LLAP_CACHE_MEM_MB[position] = LLAP_CACHE_MEM_MB[position] - LLAP_SAFETY_VALVE_MB[position] 411 | 412 | # LLAP_IO_THREADPOOL 413 | LLAP_IO_THREADPOOL[position] = LLAP_NUM_EXECUTORS_PER_DAEMON[position] * \ 414 | PERCENT_OF_EXECUTORS_FOR_IO_THREADPOOL[position] / 100 415 | run_totals_calc(position) 416 | 417 | 418 | def run_totals_calc(position): 419 | # Total LLAP Daemon Footprint 420 | # True Daemon NM Memory use needs to round to the yarn.min.container.size. 421 | # LLAP_DAEMON_CONTAINER_MEM_MB[position] 422 | # YARN_SCH_MIN_ALLOC_MEM_MB[position] 423 | # Round Up! 424 | factor = math.ceil(float(LLAP_DAEMON_CONTAINER_MEM_MB[position]) / YARN_SCH_MIN_ALLOC_MEM_MB[position]) 425 | 426 | TOTAL_LLAP_DAEMON_FOOTPRINT[position] = LLAP_NUM_NODES[position] * factor * YARN_SCH_MIN_ALLOC_MEM_MB[position] \ 427 | + LLAP_AM_DAEMON_HEAP_MB[position] 428 | 429 | # # Total LLAP Other Footprint 430 | # # TODO: Add support for PREWARMED CONTAINERS. 431 | # TOTAL_LLAP_OTHER_FOOTPRINT[position] = LLAP_CONCURRENCY[position] * TEZ_AM_MEM_MB[position] 432 | 433 | # Total LLAP Memory Footprint 434 | TOTAL_LLAP_MEM_FOOTPRINT[position] = TOTAL_LLAP_DAEMON_FOOTPRINT[position] + TOTAL_LLAP_OTHER_FOOTPRINT[position] 435 | 436 | # Total Cluster Memory Footprint 437 | TOTAL_MEM_FOOTPRINT[position] = YARN_NM_RSRC_MEM_MB[position] * WORKER_COUNT[position] 438 | 439 | # Total LLAP Yarn Queue Requirement (Percent of Root Queue) 440 | LLAP_QUEUE_MIN_REQUIREMENT[position] = round(float(TOTAL_LLAP_MEM_FOOTPRINT[position]) / 441 | TOTAL_MEM_FOOTPRINT[position] * 100, 2) 442 | calc_deltas() 443 | check_for_issues() 444 | 445 | def calc_deltas(): 446 | WORKER_MEMORY_GB[POS_DELTA[0]] = WORKER_MEMORY_GB[POS_VALUE[0]] - WORKER_MEMORY_GB[POS_CUR_VALUE[0]] 447 | WORKER_COUNT[POS_DELTA[0]] = WORKER_COUNT[POS_VALUE[0]] - WORKER_COUNT[POS_CUR_VALUE[0]] 448 | WORKER_CORES[POS_DELTA[0]] = WORKER_CORES[POS_VALUE[0]] - WORKER_CORES[POS_CUR_VALUE[0]] 449 | YARN_NM_RSRC_MEM_MB[POS_DELTA[0]] = YARN_NM_RSRC_MEM_MB[POS_VALUE[0]] - YARN_NM_RSRC_MEM_MB[POS_CUR_VALUE[0]] 450 | YARN_SCH_MAX_ALLOC_MEM_MB[POS_DELTA[0]] = YARN_SCH_MAX_ALLOC_MEM_MB[POS_VALUE[0]] - YARN_SCH_MAX_ALLOC_MEM_MB[POS_CUR_VALUE[0]] 451 | YARN_SCH_MIN_ALLOC_MEM_MB[POS_DELTA[0]] = YARN_SCH_MIN_ALLOC_MEM_MB[POS_VALUE[0]] - YARN_SCH_MIN_ALLOC_MEM_MB[POS_CUR_VALUE[0]] 452 | TOTAL_MEM_FOOTPRINT[POS_DELTA[0]] = TOTAL_MEM_FOOTPRINT[POS_VALUE[0]] - TOTAL_MEM_FOOTPRINT[POS_CUR_VALUE[0]] 453 | 454 | LLAP_NUM_NODES[POS_DELTA[0]] = LLAP_NUM_NODES[POS_VALUE[0]] - LLAP_NUM_NODES[POS_CUR_VALUE[0]] 455 | LLAP_NUM_NODES_ALT[POS_DELTA[0]] = LLAP_NUM_NODES_ALT[POS_VALUE[0]] - LLAP_NUM_NODES_ALT[POS_CUR_VALUE[0]] 456 | LLAP_CONCURRENCY[POS_DELTA[0]] = LLAP_CONCURRENCY[POS_VALUE[0]] - LLAP_CONCURRENCY[POS_CUR_VALUE[0]] 457 | TEZ_AM_MEM_MB[POS_DELTA[0]] = TEZ_AM_MEM_MB[POS_VALUE[0]] - TEZ_AM_MEM_MB[POS_CUR_VALUE[0]] 458 | LLAP_NUM_EXECUTORS_PER_DAEMON[POS_DELTA[0]] = LLAP_NUM_EXECUTORS_PER_DAEMON[POS_VALUE[0]] - LLAP_NUM_EXECUTORS_PER_DAEMON[POS_CUR_VALUE[0]] 459 | LLAP_AM_DAEMON_HEAP_MB[POS_DELTA[0]] = LLAP_AM_DAEMON_HEAP_MB[POS_VALUE[0]] - LLAP_AM_DAEMON_HEAP_MB[POS_CUR_VALUE[0]] 460 | LLAP_DAEMON_CONTAINER_MEM_MB[POS_DELTA[0]] = LLAP_DAEMON_CONTAINER_MEM_MB[POS_VALUE[0]] - LLAP_DAEMON_CONTAINER_MEM_MB[POS_CUR_VALUE[0]] 461 | LLAP_DAEMON_HEAP_MEM_MB[POS_DELTA[0]] = LLAP_DAEMON_HEAP_MEM_MB[POS_VALUE[0]] - LLAP_DAEMON_HEAP_MEM_MB[POS_CUR_VALUE[0]] 462 | LLAP_HEADROOM_MEM_MB[POS_DELTA[0]] = LLAP_HEADROOM_MEM_MB[POS_VALUE[0]] - LLAP_HEADROOM_MEM_MB[POS_CUR_VALUE[0]] 463 | LLAP_CACHE_MEM_MB[POS_DELTA[0]] = LLAP_CACHE_MEM_MB[POS_VALUE[0]] - LLAP_CACHE_MEM_MB[POS_CUR_VALUE[0]] 464 | 465 | LLAP_IO_THREADPOOL[POS_DELTA[0]] = LLAP_IO_THREADPOOL[POS_VALUE[0]] - LLAP_IO_THREADPOOL[POS_CUR_VALUE[0]] 466 | TEZ_CONTAINER_SIZE_MB[POS_DELTA[0]] = TEZ_CONTAINER_SIZE_MB[POS_VALUE[0]] - TEZ_CONTAINER_SIZE_MB[POS_CUR_VALUE[0]] 467 | LLAP_PREWARM_NUM_CONTAINERS[POS_DELTA[0]] = LLAP_PREWARM_NUM_CONTAINERS[POS_VALUE[0]] - LLAP_PREWARM_NUM_CONTAINERS[POS_CUR_VALUE[0]] 468 | TOTAL_LLAP_DAEMON_FOOTPRINT[POS_DELTA[0]] = TOTAL_LLAP_DAEMON_FOOTPRINT[POS_VALUE[0]] - TOTAL_LLAP_DAEMON_FOOTPRINT[POS_CUR_VALUE[0]] 469 | TOTAL_LLAP_OTHER_FOOTPRINT[POS_DELTA[0]] = TOTAL_LLAP_OTHER_FOOTPRINT[POS_VALUE[0]] - TOTAL_LLAP_OTHER_FOOTPRINT[POS_CUR_VALUE[0]] 470 | TOTAL_LLAP_MEM_FOOTPRINT[POS_DELTA[0]] = TOTAL_LLAP_MEM_FOOTPRINT[POS_VALUE[0]] - TOTAL_LLAP_MEM_FOOTPRINT[POS_CUR_VALUE[0]] 471 | LLAP_QUEUE_MIN_REQUIREMENT[POS_DELTA[0]] = LLAP_QUEUE_MIN_REQUIREMENT[POS_VALUE[0]] - LLAP_QUEUE_MIN_REQUIREMENT[POS_CUR_VALUE[0]] 472 | 473 | def check_for_issues(): 474 | del ISSUE_MESSAGES[:] 475 | message2 = [RECOMMENDATION_TYPE, 476 | "LLAP will flex above YARN container boundaries for a very short time " + 477 | "under highload/join scenarios and may cause YARN to " + 478 | "prematurely KILL LLAP Daemon containers.", 479 | ["In yarn-site.xml, set 'yarn.nodemanager.pmem-check-enabled=false'", 480 | "Apply this only to nodes used to run LLAP daemons", 481 | "Use a Node Label, Queue, and Managed Groups in Ambari to control.", 482 | "Setting is used by the Node Manager"]] 483 | ISSUE_MESSAGES.append(message2) 484 | 485 | if WORKER_CORES[POS_VALUE[0]] < 1: 486 | messageW = [ERROR_TYPE, WORKER_CORES[POS_SHORT_DESC[0]] + " hasn't been set.", 487 | ["Set workers cores to run calculator."]] 488 | ISSUE_MESSAGES.append(messageW) 489 | if WORKER_COUNT[POS_VALUE[0]] < 1: 490 | messageC = [ERROR_TYPE, WORKER_COUNT[POS_SHORT_DESC[0]] + " hasn't been set.", 491 | ["Set workers count to run calculator."]] 492 | ISSUE_MESSAGES.append(messageC) 493 | if WORKER_MEMORY_GB[POS_VALUE[0]] < 1: 494 | messageM = [ERROR_TYPE, WORKER_MEMORY_GB[POS_SHORT_DESC[0]] + " hasn't been set.", 495 | ["Set workers memory to run calculator."]] 496 | ISSUE_MESSAGES.append(messageM) 497 | if LLAP_NUM_NODES[POS_VALUE[0]] < 1: 498 | messageLN = [ERROR_TYPE, LLAP_NUM_NODES[POS_SHORT_DESC[0]] + " hasn't been set.", 499 | ["Set num of LLAP Nodes to run calculator."]] 500 | ISSUE_MESSAGES.append(messageLN) 501 | if LLAP_CONCURRENCY[POS_VALUE[0]] < 1: 502 | messageLC = [ERROR_TYPE, LLAP_CONCURRENCY[POS_SHORT_DESC[0]] + " hasn't been set.", 503 | ["Set concurrency to run calculator."]] 504 | ISSUE_MESSAGES.append(messageLC) 505 | 506 | if LLAP_DAEMON_HEAP_MEM_MB[POS_VALUE[0]] > LLAP_DAEMON_CONTAINER_MEM_MB[POS_VALUE[0]]: 507 | message = [ERROR_TYPE, LLAP_DAEMON_CONTAINER_MEM_MB[POS_SHORT_DESC[0]] + ":" + 508 | str(LLAP_DAEMON_CONTAINER_MEM_MB[POS_VALUE[0]]) + 509 | " can't be less than " + LLAP_DAEMON_HEAP_MEM_MB[POS_SHORT_DESC[0]] + ":" + 510 | str(LLAP_DAEMON_HEAP_MEM_MB[POS_VALUE[0]]), 511 | ["Decrease " + LLAP_NUM_EXECUTORS_PER_DAEMON[POS_SHORT_DESC[0]], 512 | "Decrease " + LLAP_TASK_MB_PER_INSTANCE_REFERENCE[POS_SHORT_DESC[0]]]] 513 | ISSUE_MESSAGES.append(message) 514 | if LLAP_DAEMON_CONTAINER_MEM_MB[POS_VALUE[0]] > LLAP_DAEMON_CONTAINER_SAFETY_GB[POS_VALUE[0]] * GB: 515 | message3 = [RULE_APPLICATION_TYPE, LLAP_DAEMON_CONTAINER_MEM_MB[POS_SHORT_DESC[0]] + ":" + 516 | str(LLAP_DAEMON_CONTAINER_MEM_MB[POS_VALUE[0]]) + 517 | " is greater than " + str(LLAP_DAEMON_CONTAINER_SAFETY_GB[POS_VALUE[0]]) + 518 | "Gb which has implications on memory " + 519 | "and may cause YARN to prematurely KILL LLAP Daemon containers", 520 | ["Therefore, we've applied a 'Safety Value' threshold to the total memory footprint of the LLAP daemon.", 521 | str(LLAP_SAFETY_VALVE_MB[POS_VALUE[0]]) + "Mb was subtracted from the cache" 522 | ]] 523 | ISSUE_MESSAGES.append(message3) 524 | if ((LLAP_TASK_MB_PER_INSTANCE_REFERENCE[POS_VALUE[0]] * LLAP_NUM_EXECUTORS_PER_DAEMON[POS_VALUE[0]] * 1.5) < LLAP_DAEMON_HEAP_MEM_MB[POS_VALUE[0]]): 525 | message4 = [RULE_APPLICATION_TYPE, LLAP_DAEMON_HEAP_MEM_MB[POS_SHORT_DESC[0]] + ":" + 526 | str(LLAP_DAEMON_HEAP_MEM_MB[POS_VALUE[0]]) + 527 | " is greater than 150% of:\n\t\t- " + 528 | LLAP_TASK_MB_PER_INSTANCE_REFERENCE[POS_SHORT_DESC[0]] + ":[" + 529 | str(LLAP_TASK_MB_PER_INSTANCE_REFERENCE[POS_VALUE[0]]) + "] * " + 530 | LLAP_NUM_EXECUTORS_PER_DAEMON[POS_SHORT_DESC[0]] + ":[" + 531 | str(LLAP_NUM_EXECUTORS_PER_DAEMON[POS_VALUE[0]]) + "] (" + 532 | str((LLAP_TASK_MB_PER_INSTANCE_REFERENCE[POS_VALUE[0]] * LLAP_NUM_EXECUTORS_PER_DAEMON[POS_VALUE[0]] * 1.5)) + ")" + 533 | ".\n\t\tThis might indicate an imbalance of cores and memory.", 534 | ["Consider increasing 'executors' without over extending cores.", 535 | "Consider increasing 'cache percentage' to adjust the imbalance.", 536 | "Do nothing, because your queries need a bigger footprint." 537 | ]] 538 | ISSUE_MESSAGES.append(message4) 539 | 540 | 541 | def get_current(selection, lst): 542 | for item in lst: 543 | if item[0] == selection: 544 | return item[2] 545 | 546 | 547 | def set_value(selection, lst): 548 | current = [] 549 | for item in lst: 550 | if item[0] == selection: 551 | current = item 552 | 553 | new_value = raw_input("{0}\t[{1}]:".format(current[1], current[2])) 554 | 555 | current[2] = new_value 556 | 557 | 558 | def convert(value, original): 559 | if isinstance(original, int): 560 | return int(value) 561 | if isinstance(original, long): 562 | return long(value) 563 | if isinstance(original, str): 564 | return str(value) 565 | 566 | 567 | def filtered_sections(): 568 | f_sections = [] 569 | for section in SECTIONS: 570 | for config in LOGICAL_CONFIGS: 571 | if config[POS_SECTION[0]] == section and config[POS_TYPE[0]] in MODE: 572 | f_sections.append(section) 573 | break 574 | 575 | return f_sections 576 | 577 | 578 | def sections_loop(): 579 | while True: 580 | # List Sections 581 | # print ("sections") 582 | inc = 1 583 | for section in filtered_sections(): 584 | print (" {0} - {1}".format(inc, section[0])) 585 | inc += 1 586 | 587 | # Select Section 588 | try: 589 | raw_selection = raw_input(SELECT_SECTION) 590 | if raw_selection == "": 591 | break 592 | selection = int(raw_selection) 593 | except ValueError: 594 | # print (GO_BACK) 595 | return 596 | 597 | # Check selection boundaries 598 | if selection > len(filtered_sections()) or selection < 1: 599 | # print (GO_BACK) 600 | return 601 | 602 | if not section_loop(selection): 603 | break 604 | 605 | 606 | def section_loop(selection): 607 | while True: 608 | # Identify Section 609 | # print ("section") 610 | section_choice = "" 611 | inc = 1 612 | for section in filtered_sections(): 613 | if selection == inc: 614 | section_choice = section 615 | inc += 1 616 | 617 | # Find configs in Section that are "TYPE_INPUT" 618 | section_configs = [] 619 | for config in LOGICAL_CONFIGS: 620 | if isinstance(config[POS_SECTION[0]], tuple) and config[POS_SECTION[0]][1] == section_choice[1] and config[POS_TYPE[0]] in MODE: 621 | section_configs.append(config) 622 | 623 | config = select_config(section_choice, section_configs) 624 | 625 | if not config: 626 | return True 627 | 628 | if not change_config(config): 629 | return True 630 | else: 631 | run_calc(POS_VALUE[0]) 632 | 633 | 634 | def select_config(section, section_configs): 635 | # List Filtered Sections 636 | # print ("select") 637 | print (chr(27) + "[2J") 638 | print ("===================================") 639 | print (" " + section[0]) 640 | print ("===================================") 641 | inc = 1 642 | for config in section_configs: 643 | print (" {0} - {1}: [{2}]".format(inc, config[POS_SHORT_DESC[0]], config[POS_VALUE[0]])) 644 | inc += 1 645 | 646 | print (ENTER_RETURN) 647 | print ("===================================") 648 | # Pick Config to Edit 649 | try: 650 | choice = int(raw_input(SELECT_CONFIG)) 651 | except ValueError: 652 | # print (GO_BACK) 653 | return True 654 | 655 | # Check selection boundaries 656 | if choice > (inc) or choice < 1: 657 | # print (GO_BACK) 658 | return True 659 | 660 | # Enter New Value 661 | inc = 1 662 | # target_config = [] 663 | for config in section_configs: 664 | if choice == inc: 665 | return config 666 | inc += 1 667 | 668 | 669 | def change_config(config): 670 | # print ("change") 671 | 672 | try: 673 | raw_value = raw_input(">>> {0} \"{1}\" [{2}]: ".format(config[POS_SHORT_DESC[0]], config[POS_CONFIG[0]], config[POS_VALUE[0]])) 674 | 675 | if raw_value == "": 676 | return True 677 | new_value = convert(raw_value, config[POS_VALUE[0]]) 678 | except: 679 | # print ("Error Setting Value, try again. Most likely a bad conversion.") 680 | return False 681 | 682 | config[POS_VALUE[0]] = new_value 683 | 684 | return True 685 | 686 | 687 | def guided_loop(): 688 | # Find configs in Section that are "TYPE_INPUT" 689 | print(chr(27) + "[2J") 690 | print ("") 691 | environment_status() 692 | print ("===================================") 693 | print (" Guided Configuration ") 694 | print ("") 695 | print ("- Enter value for each setting.") 696 | print ("- Press 'enter' to keep current.") 697 | print ("") 698 | print ("===================================") 699 | 700 | guided_configs = [] 701 | for config in LOGICAL_CONFIGS: 702 | if config[POS_TYPE[0]] in MODE: 703 | guided_configs.append(config) 704 | 705 | for config in guided_configs: 706 | change_config(config) 707 | 708 | run_calc(POS_VALUE[0]) 709 | logical_display() 710 | 711 | 712 | def edit_loop(): 713 | while True: 714 | print(chr(27) + "[2J") 715 | print ("") 716 | environment_status() 717 | print ("===================================") 718 | print (" Edit Configurations ") 719 | print ("===================================") 720 | 721 | # List Sections 722 | # print ("sections") 723 | inc = 1 724 | for section in filtered_sections(): 725 | print (" {0} - {1}".format(inc, section[0])) 726 | inc += 1 727 | 728 | print (ENTER_RETURN) 729 | print ("===================================") 730 | # Select Section 731 | try: 732 | raw_selection = raw_input(SELECT_SECTION) 733 | if raw_selection == "": 734 | break 735 | selection = int(raw_selection) 736 | except ValueError: 737 | # print (GO_BACK) 738 | return 739 | 740 | # Check selection boundaries 741 | if selection > len(filtered_sections()) or selection < 1: 742 | # print (GO_BACK) 743 | return 744 | 745 | if not section_loop(selection): 746 | break 747 | 748 | print ("===================================") 749 | 750 | 751 | def logical_display(): 752 | run_calc(POS_VALUE[0]) 753 | 754 | global LOGICAL_CONFIGS 755 | pprinttable(LOGICAL_CONFIGS, DISPLAY_COLUMNS) 756 | 757 | print ("") 758 | environment_status() 759 | print ("") 760 | raw_input("press enter...") 761 | 762 | def ambari_configs(): 763 | run_calc(POS_VALUE[0]) 764 | print(chr(27) + "[2J") 765 | print ("===================================") 766 | print (" Ambari Configurations ") 767 | print ("===================================") 768 | 769 | pprinttable(AMBARI_CONFIGS, DISPLAY_COLUMNS) 770 | 771 | ambaricalls = ambariRestCalls() 772 | print ("===================================") 773 | print (" Ambari REST Call Configurations ") 774 | print ("===================================") 775 | if ISSUE_MESSAGES > 0: 776 | environment_status() 777 | else: 778 | for line in ambaricalls: 779 | print line 780 | 781 | manual = manualCfgs() 782 | if len(manual) > 0: 783 | print ("===================================") 784 | print (" Manual Configurations ") 785 | print ("===================================") 786 | 787 | for cfg in manual: 788 | print ("Manual Configuration: {0} [{1}]".format(cfg[POS_SHORT_DESC[0]], cfg[POS_VALUE[0]])) 789 | 790 | print ("") 791 | raw_input("press enter...") 792 | 793 | def manualCfgs(): 794 | manual = [] 795 | for cfg in AMBARI_CONFIGS: 796 | if cfg[POS_SECTION[0]] not in VALID_AMBARI_SECTIONS: 797 | manual.append(cfg) 798 | 799 | return manual 800 | 801 | def ambariRestCalls(version_note): 802 | ambariConfigs = [] 803 | for cfg in AMBARI_CONFIGS: 804 | if cfg[POS_SECTION[0]] in VALID_AMBARI_SECTIONS: 805 | if len(version_note) > 0: 806 | ambariConfigs.append(AMBARI_CFG_CMD_V.format(SSL_CMD, cfg[POS_SECTION[0]][1], cfg[POS_CONFIG[0]], cfg[POS_VALUE[0]], version_note)) 807 | else: 808 | ambariConfigs.append(AMBARI_CFG_CMD.format(SSL_CMD, cfg[POS_SECTION[0]][1], cfg[POS_CONFIG[0]], cfg[POS_VALUE[0]])) 809 | 810 | return ambariConfigs 811 | 812 | 813 | def save(): 814 | lclErrors = getIssues() 815 | hasError = False 816 | for message in ISSUE_MESSAGES: 817 | if message[0] == ERROR_TYPE: 818 | hasError = True 819 | 820 | if hasError: 821 | environment_status() 822 | print ("Fix 'ERRORS' to get script output!") 823 | print (raw_input(ENTER_CONTINUE)) 824 | else: 825 | out_file_base = raw_input("Enter Filename(without Extension):") 826 | t = datetime.datetime.now() 827 | version_note = "LLAP Config from CALC v." + VERSION + " : " + t.strftime('%Y-%m-%d %H:%M:%S') 828 | note_choice = raw_input("Enter version notes for AMBARI REST Calls: [" + version_note + "]") 829 | if len(note_choice) > 0: 830 | version_note = note_choice 831 | 832 | # Remove Quotes 833 | version_note = version_note.replace("\"","") 834 | 835 | myFile = open(out_file_base + ".sh", "w") 836 | 837 | myFile.write("export AMBARI_HOST=\n") 838 | myFile.write("export AMBARI_PORT=\n") 839 | myFile.write("export CLUSTER_NAME=\n") 840 | 841 | for line in ambariRestCalls(version_note): 842 | myFile.write(line) 843 | myFile.write('\n') 844 | 845 | myFile.close() 846 | 847 | myFile = open(out_file_base + ".txt", "w") 848 | 849 | # myFile.writelines(buildtable(AMBARI_CONFIGS, DISPLAY_COLUMNS)) 850 | for line in buildtable(LOGICAL_CONFIGS, DISPLAY_COLUMNS): 851 | myFile.write(line) 852 | myFile.write('\n') 853 | 854 | manual = manualCfgs() 855 | if len(manual) > 0: 856 | for line in manual: 857 | myFile.writelines("Manual Configuration: {0} [{1}]".format(line[POS_SHORT_DESC[0]], line[POS_VALUE[0]])) 858 | myFile.write('\n') 859 | 860 | myFile.write("\n") 861 | myFile.write(" Calc Version:\t" + VERSION + "\n") 862 | myFile.write("Ambari Integration On:\t("+str(ambari_integration)+")\n") 863 | myFile.write(" Current mode:\t*"+str(MODE)+"*\n") 864 | myFile.write(" Display Columns:\t" + str(getDisplayColumns()) + "\n") 865 | myFile.write("\n") 866 | lclIssues = getIssues() 867 | if len(lclIssues) > 0: 868 | for line in lclIssues: 869 | myFile.write(line + "\n") 870 | 871 | myFile.close() 872 | print ("") 873 | print ("Saved to: " + out_file_base + ".txt - Configuration Grid") 874 | print ("Saved to: " + out_file_base + ".sh - Ambari Configuration REST Script") 875 | print ("") 876 | print (raw_input(ENTER_CONTINUE)) 877 | 878 | def getDisplayColumns(): 879 | rtn = [] 880 | for item in DISPLAY_COLUMNS: 881 | rtn.append(item[1]) 882 | return rtn 883 | 884 | 885 | def change_mode(): 886 | while True: 887 | print(chr(27) + "[2J") 888 | print ("") 889 | environment_status() 890 | print ("===================================") 891 | print (" Change Mode ") 892 | print ("===================================") 893 | print (" 1 - Simple Mode") 894 | print (" 2 - Reference Mode(expose additional settings)") 895 | print ("") 896 | print ("===================================") 897 | print (" Toggle Columns for Display") 898 | print ("===================================") 899 | print (" 3 - Short Desc") 900 | print (" 4 - Type") 901 | print (" 5 - Section") 902 | print (" 6 - Config") 903 | print (" 7 - Value") 904 | print (" 8 - Current Value") 905 | print (" 9 - Options") 906 | print (" 10 - Long Desc") 907 | print (" 11 - Delta") 908 | print ("===================================") 909 | print (ENTER_RETURN) 910 | 911 | try: 912 | raw_selection = raw_input(SELECT_MODE) 913 | if raw_selection == "": 914 | break 915 | selection = int(raw_selection) 916 | except ValueError: 917 | break 918 | 919 | if selection is not None and selection in (1,2): 920 | if selection == 1: 921 | if TYPE_REFERENCE in MODE: 922 | MODE.remove(TYPE_REFERENCE) 923 | else: 924 | if TYPE_REFERENCE not in MODE: 925 | MODE.append(TYPE_REFERENCE) 926 | elif selection is not None and selection in (3,4,5,6,7,8,9,10,11): 927 | if (selection == 8 or selection == 11) and current_values == False: 928 | break 929 | selectionadjusted = selection - 3 930 | for i in ALL_DISPLAY_COLUMNS: 931 | if i[0] == selectionadjusted: 932 | if len(DISPLAY_COLUMNS) == 0: 933 | DISPLAY_COLUMNS.append(i) 934 | elif i in DISPLAY_COLUMNS: 935 | DISPLAY_COLUMNS.remove(i) 936 | else: 937 | spot = i[0] 938 | iter = 0 939 | for i2 in DISPLAY_COLUMNS: 940 | if i2[0] > spot: 941 | DISPLAY_COLUMNS.insert(iter,i) 942 | break 943 | else: 944 | iter += 1 945 | if iter >= len(DISPLAY_COLUMNS): 946 | DISPLAY_COLUMNS.insert(iter,i) 947 | break 948 | else: 949 | break 950 | print ("________________________________") 951 | print ("") 952 | 953 | def getIssues(): 954 | ISSUES = [] 955 | if len(ISSUE_MESSAGES) > 0: 956 | ISSUES.append ("********************* ISSUES *********************") 957 | for message in ISSUE_MESSAGES: 958 | ISSUES.append ("Type : " + message[0]) 959 | ISSUES.append ("Issue: ") 960 | ISSUES.append ("\t\t" + message[1]) 961 | ISSUES.append ("Options:") 962 | for opt in message[2]: 963 | ISSUES.append ("\t\t" + opt) 964 | ISSUES.append(" ------------------------") 965 | ISSUES.append ("**************************************************") 966 | return ISSUES 967 | 968 | 969 | def environment_status(): 970 | print (" Calc Version:\t" + VERSION) 971 | print ("Ambari Integration On:\t("+str(ambari_integration)+")") 972 | print (" Current Values On:\t("+str(current_values)+")") 973 | print (" Current mode:\t*"+str(MODE)+"*") 974 | print (" Display Columns:\t" + str(getDisplayColumns())) 975 | lclIssues = getIssues() 976 | if len(lclIssues) > 0: 977 | for line in lclIssues: 978 | print (line) 979 | 980 | def action_loop(): 981 | actions = ( 982 | ("Guided Config", "g"), (), ("Logical Display", "l"), ("Ambari Config List", "a"), (), ("Edit", "e"), 983 | ("Save", "s"), 984 | (), ("Mode (expose additional settings)", "m"), (), ("Quit", "q")) 985 | 986 | def validate(choice): 987 | for action in actions: 988 | if len(action) > 1 and choice == action[1]: 989 | return True 990 | print(chr(27) + "[2J") 991 | print ("") 992 | environment_status() 993 | print ("===================================") 994 | print (" MAIN Action Menu ") 995 | print ("===================================") 996 | for action in actions: 997 | if len(action)>1: 998 | print (" {1} - {0}".format(action[0], action[1])) 999 | else: 1000 | print (" -----------") 1001 | print ("===================================") 1002 | 1003 | selection = raw_input(SELECT_ACTION) 1004 | 1005 | if validate(selection): 1006 | # print("Good choice %s" % selection) 1007 | if selection == "q": 1008 | return False 1009 | elif selection == "g": 1010 | guided_loop() 1011 | return True 1012 | elif selection == "e": 1013 | edit_loop() 1014 | return True 1015 | elif selection == "l": 1016 | logical_display() 1017 | return True 1018 | elif selection == "a": 1019 | ambari_configs() 1020 | return True 1021 | elif selection == "s": 1022 | save() 1023 | return True 1024 | elif selection == "m": 1025 | change_mode() 1026 | return True 1027 | else: 1028 | return True 1029 | else: 1030 | print("No Good, try again.") 1031 | return True 1032 | 1033 | 1034 | def populate_ambari_rest_current(): 1035 | if not ambari_integration: 1036 | return False 1037 | 1038 | section_configs = {} 1039 | for configs in VALID_AMBARI_SECTIONS: 1040 | with Capturing() as output: 1041 | get_properties(cluster, configs[1], [], ambari_accessor_api) 1042 | lclJson = "".join(output) 1043 | section_configs[configs[1]] = json.loads(lclJson) 1044 | 1045 | populate_current(section_configs) 1046 | 1047 | 1048 | def populate_ambari_bp_current( blueprintFile ): 1049 | blueprint = json.loads(open (blueprintFile).read()) 1050 | bp_configs = blueprint['configurations'] 1051 | 1052 | section_configs = {} 1053 | 1054 | for section in VALID_AMBARI_SECTIONS: 1055 | 1056 | for config in bp_configs: 1057 | for key, value in config.items(): 1058 | if key == section[1]: 1059 | # print "got properties for: " + key 1060 | section_configs[section[1]] = value 1061 | # else: 1062 | # print "next" 1063 | 1064 | populate_current(section_configs) 1065 | 1066 | 1067 | def populate_current( section_configs ): 1068 | for scKey in section_configs.keys(): 1069 | section_config = section_configs[scKey] 1070 | 1071 | for configs in VALID_AMBARI_SECTIONS: 1072 | for ambariConfig in AMBARI_CONFIGS: 1073 | if ambariConfig[POS_SECTION[0]][1] == scKey: 1074 | try: 1075 | # set_config(ambariConfig, POS_CUR_VALUE[0]) 1076 | ambariConfig[POS_CUR_VALUE[0]] = convert(section_config['properties'][ambariConfig[POS_CONFIG[0]]], ambariConfig[POS_CUR_VALUE[0]]) 1077 | # Set Calc Values to the same. 1078 | ambariConfig[POS_VALUE[0]] = convert(section_config['properties'][ambariConfig[POS_CONFIG[0]]], ambariConfig[POS_CUR_VALUE[0]]) 1079 | except Exception as e: 1080 | logger.debug(e) 1081 | logger.debug("Skipping property lookup: " + str(ambariConfig[POS_CONFIG[0]])) 1082 | 1083 | if LLAP_NUM_NODES[POS_CUR_VALUE[0]] != LLAP_NUM_NODES_ALT[POS_CUR_VALUE[0]]: 1084 | print ("WARNING: In your current Ambari Configuration, similar legacy configurations are not in Sync. These need to be in sync!!!!\n\t" + 1085 | LLAP_NUM_NODES[POS_CONFIG[0]] + ":" + str(LLAP_NUM_NODES[POS_CUR_VALUE[0]]) + "\n\t" + 1086 | LLAP_NUM_NODES_ALT[POS_CONFIG[0]] + ":" + str(LLAP_NUM_NODES_ALT[POS_CUR_VALUE[0]]) + 1087 | "\nOur calculations for the current configuration may be off until these are corrected.") 1088 | raw_input("press enter to continue...") 1089 | 1090 | run_totals_calc(POS_CUR_VALUE[0]) 1091 | # Reset the Calc Values based on initial Ambari Values. 1092 | run_calc(POS_VALUE[0]) 1093 | 1094 | 1095 | def main(): 1096 | global ambari_integration 1097 | global current_values 1098 | global cluster 1099 | global version_note 1100 | global ambari_accessor_api 1101 | 1102 | parser = optparse.OptionParser(usage="usage: %prog [options]") 1103 | 1104 | login_options_group = OptionGroup(parser, "To specify credentials please use \"-e\" OR \"-u\" and \"-p'\"") 1105 | login_options_group.add_option("-u", "--user", dest="user", default="admin", help="Optional user ID to use for authentication. Default is 'admin'") 1106 | login_options_group.add_option("-p", "--password", dest="password", default="admin", help="Optional password to use for authentication. Default is 'admin'") 1107 | login_options_group.add_option("-e", "--credentials-file", dest="credentials_file", help="Optional file with user credentials separated by new line.") 1108 | parser.add_option_group(login_options_group) 1109 | 1110 | parser.add_option("-t", "--port", dest="port", default="8080", help="Optional port number for Ambari server. Default is '8080'. Provide empty string to not use port.") 1111 | parser.add_option("-s", "--protocol", dest="protocol", default="http", help="Optional support of SSL. Default protocol is 'http'") 1112 | parser.add_option("--unsafe", action="store_true", dest="unsafe", help="Skip SSL certificate verification.") 1113 | 1114 | parser.add_option("-l", "--host", dest="host", help="Server external host name") 1115 | parser.add_option("-n", "--cluster", dest="cluster", help="Name given to cluster. Ex: 'c1'") 1116 | 1117 | parser.add_option("-v", "--version-note", dest="version_note", default="", help="Version change notes which will help to know what has been changed in this config. This value is optional and is used for actions and .") 1118 | 1119 | parser.add_option("-w", "--workers", dest="workers", help="How many worker nodes in the cluster?") 1120 | parser.add_option("-m", "--memory", dest="memory", help="How much memory does each worker node have (GB)?") 1121 | 1122 | parser.add_option("-b", "--ambari-blueprint", dest="ambari_blueprint", help="Use an Ambari Blueprint to pull configs.") 1123 | 1124 | (options, args) = parser.parse_args() 1125 | 1126 | logger.setLevel(logging.INFO) 1127 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') 1128 | stdout_handler = logging.StreamHandler(sys.stdout) 1129 | stdout_handler.setLevel(logging.INFO) 1130 | stdout_handler.setFormatter(formatter) 1131 | logger.addHandler(stdout_handler) 1132 | 1133 | global SSL_CMD 1134 | 1135 | # options with default value 1136 | if options.protocol: 1137 | SSL_CMD += "-s " 1138 | SSL_CMD += options.protocol 1139 | 1140 | if options.unsafe: 1141 | SSL_CMD += " --unsafe" 1142 | 1143 | if not options.credentials_file and (not options.user or not options.password): 1144 | ambari_integration = False 1145 | logger.info("Ambari Credential information missing. Running in standalone mode.") 1146 | # parser.error("You should use option (-e) to set file with Ambari user credentials" 1147 | # " OR use (-u) username and (-p) password") 1148 | 1149 | if options.credentials_file: 1150 | if os.path.isfile(options.credentials_file): 1151 | try: 1152 | with open(options.credentials_file) as credentials_file: 1153 | file_content = credentials_file.read() 1154 | login_lines = filter(None, file_content.splitlines()) 1155 | if len(login_lines) == 2: 1156 | user = login_lines[0] 1157 | password = login_lines[1] 1158 | else: 1159 | logger.error("Incorrect content of {0} file. File should contain Ambari username and password separated by new line.".format(options.credentials_file)) 1160 | return -1 1161 | except Exception as e: 1162 | logger.error("You don't have permissions to {0} file".format(options.credentials_file)) 1163 | return -1 1164 | else: 1165 | logger.error("File {0} doesn't exist or you don't have permissions.".format(options.credentials_file)) 1166 | return -1 1167 | else: 1168 | user = options.user 1169 | password = options.password 1170 | 1171 | port = options.port 1172 | protocol = options.protocol 1173 | 1174 | if options.workers: 1175 | WORKER_COUNT[POS_VALUE[0]] = int(options.workers) 1176 | WORKER_COUNT[POS_CUR_VALUE[0]] = int(options.workers) 1177 | if options.memory: 1178 | WORKER_MEMORY_GB[POS_VALUE[0]] = int(options.memory) 1179 | WORKER_MEMORY_GB[POS_CUR_VALUE[0]] = int(options.memory) 1180 | 1181 | #options without default value 1182 | if None in [options.host, options.cluster]: 1183 | ambari_integration = False 1184 | current_values = False 1185 | logger.info("Ambari Integration information missing. Running in standalone mode.") 1186 | # parser.error("One of required options is not passed") 1187 | 1188 | # action = options.action 1189 | host = options.host 1190 | cluster = options.cluster 1191 | # config_type = options.config_type 1192 | version_note = options.version_note 1193 | 1194 | if ambari_integration: 1195 | ambari_accessor_api = api_accessor(host, user, password, protocol, port, options.unsafe) 1196 | populate_ambari_rest_current() 1197 | DISPLAY_COLUMNS.append(POS_CUR_VALUE) 1198 | 1199 | if options.ambari_blueprint and not ambari_integration: 1200 | current_values = True 1201 | populate_ambari_bp_current(options.ambari_blueprint) 1202 | DISPLAY_COLUMNS.append(POS_CUR_VALUE) 1203 | if None in [options.workers, options.memory]: 1204 | logger.info("** Include Worker Count (-w) and Worker Memory (-m) for comprehensive settings when providing a Blueprint (-b)") 1205 | return 1206 | 1207 | if not current_values: 1208 | guided_loop() 1209 | 1210 | # Setup Base defaults 1211 | run_calc(POS_VALUE[0]) 1212 | 1213 | while True: 1214 | if not action_loop(): 1215 | break 1216 | 1217 | 1218 | main() 1219 | -------------------------------------------------------------------------------- /layout_rpt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # use this to parse the Ambari Layout Report that's generated with: 4 | # http://${AMBARI_HOST_PORT}/api/v1/clusters/${CLUSTER_NAME}/hosts?fields=Hosts/host_name,host_components,Hosts/ip,Hosts/total_mem,Hosts/os_arch,Hosts/os_type,Hosts/rack_info,Hosts/cpu_count,Hosts/disk_info,metrics/disk,Hosts/ph_cpu_count 5 | 6 | import optparse 7 | import logging 8 | import sys 9 | import json 10 | from common import pprinttable 11 | 12 | logger = logging.getLogger('LLAPConfig') 13 | 14 | def get_hostname( item ): 15 | host_info = item["Hosts"] 16 | return host_info["host_name"] 17 | 18 | 19 | def is_component( item, componentName ): 20 | components = item["host_components"] 21 | for component in components: 22 | for ckey, cvalue in component.items(): 23 | if ckey == "HostRoles": 24 | for hkey, hvalue in cvalue.items(): 25 | if hkey == "component_name": 26 | if hvalue == componentName: 27 | return True 28 | return False 29 | 30 | 31 | def get_info(layoutFile): 32 | layout = json.loads(open(layoutFile).read()) 33 | items = layout['items'] 34 | 35 | hosttable, compute_count, other_count = gen_hosttable( items ) 36 | 37 | return hosttable, compute_count, other_count 38 | 39 | 40 | def report(layoutFile): 41 | layout = json.loads(open(layoutFile).read()) 42 | items = layout['items'] 43 | 44 | hosttable, compute_count, other_count = gen_hosttable( items ) 45 | 46 | rpt_hosttable(hosttable) 47 | rpt_count_type('Compute', compute_count) 48 | rpt_count_type('Other', other_count) 49 | rpt_totals(hosttable) 50 | 51 | def gen_hosttable( items ): 52 | records = [] 53 | compute_count = {} 54 | other_count = {} 55 | 56 | for item in items: 57 | record = [] 58 | host = item["Hosts"] 59 | record.append(host["host_name"]) 60 | record.append(host["cpu_count"]) 61 | record.append(host["os_type"]) 62 | record.append(host["total_mem"] / (1024 * 1024)) 63 | record.append(host["rack_info"]) 64 | 65 | record.append(is_component(item, "DATANODE")) 66 | record.append(is_component(item, "NODEMANAGER")) 67 | records.append(record) 68 | compute = is_component(item, "NODEMANAGER") 69 | key = str(compute) + str(record[3]) + str(record[1]) 70 | memory = record[3] 71 | cores = record[1] 72 | if compute and key not in compute_count: 73 | compute_count[key] = {'count': 1, 'memory': memory, 'cores': cores, } 74 | elif compute: 75 | compute_count[key]['count'] += 1 76 | elif not compute and key not in other_count: 77 | other_count[key] = {'count': 1, 'memory': memory, 'cores': cores, } 78 | elif not compute: 79 | other_count[key]['count'] += 1 80 | 81 | # print key + str(memory) + str(cores) 82 | 83 | return records, compute_count, other_count 84 | 85 | def rpt_hosttable ( hosttable ): 86 | # master = datanode & compute 87 | fields = [[0, 'Host'], [1, 'CPU Count'], [2, 'OS'], [3, 'Memory'], [4, 'Rack'], [5, 'Data Node'], [6, 'Compute Node']] 88 | pprinttable(hosttable, fields) 89 | 90 | def rpt_count_type ( type, count_type ): 91 | # master = datanode & compute 92 | print type 93 | fields = [[0, 'Count'], [1, 'Memory'], [2, 'Cores']] 94 | count_type_rows = [] 95 | for key in count_type: 96 | count_type_record = [count_type[key]['count'], count_type[key]['memory'], count_type[key]['cores']] 97 | count_type_rows.append(count_type_record) 98 | 99 | pprinttable(count_type_rows, fields) 100 | 101 | 102 | def rpt_totals ( hosttable ): 103 | totalFields = [[0,"Type"],[1,"Count"],[2, "OS"],[3,"CPU-Min"], [4,"CPU-Max"], [5,"Mem-Min"],[6,"Mem-Max"]] 104 | totalType = [] 105 | 106 | datanodes = ["Data Nodes", 0, [], 10000, 0, 100000, 0] 107 | for record in hosttable: 108 | if record[5]: 109 | datanodes[1] += 1 110 | if (record[2] not in datanodes[2]): 111 | datanodes[2].append(record[2]) 112 | # CPU Min 113 | if record[1] < datanodes[3]: 114 | datanodes[3] = record[1] 115 | # CPU Max 116 | if record[1] > datanodes[4]: 117 | datanodes[4] = record[1] 118 | # Mem Min 119 | if record[3] < datanodes[5]: 120 | datanodes[5] = record[3] 121 | # Mem Max 122 | if record[3] > datanodes[6]: 123 | datanodes[6] = record[3] 124 | 125 | totalType.append(datanodes) 126 | 127 | computeNodes = ["Compute Nodes", 0, [], 10000, 0, 100000, 0] 128 | for record in hosttable: 129 | if record[6]: 130 | computeNodes[1] += 1 131 | if (record[2] not in computeNodes[2]): 132 | computeNodes[2].append(record[2]) 133 | # CPU Min 134 | if record[1] < computeNodes[3]: 135 | computeNodes[3] = record[1] 136 | # CPU Max 137 | if record[1] > computeNodes[4]: 138 | computeNodes[4] = record[1] 139 | # Mem Min 140 | if record[3] < computeNodes[5]: 141 | computeNodes[5] = record[3] 142 | # Mem Max 143 | if record[3] > computeNodes[6]: 144 | computeNodes[6] = record[3] 145 | 146 | totalType.append(computeNodes) 147 | 148 | pprinttable(totalType, totalFields) 149 | 150 | 151 | def main(): 152 | # global ambari_integration 153 | global cluster 154 | # global version_note 155 | # global ambari_accessor_api 156 | 157 | parser = optparse.OptionParser(usage="usage: %prog [options]") 158 | 159 | parser.add_option("-l", "--ambari-layout", dest="ambari_layout", help=".") 160 | 161 | (options, args) = parser.parse_args() 162 | 163 | logger.setLevel(logging.INFO) 164 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') 165 | stdout_handler = logging.StreamHandler(sys.stdout) 166 | stdout_handler.setLevel(logging.INFO) 167 | stdout_handler.setFormatter(formatter) 168 | logger.addHandler(stdout_handler) 169 | 170 | if options.ambari_layout: 171 | report(options.ambari_layout) 172 | 173 | main() 174 | --------------------------------------------------------------------------------