├── .gitignore ├── LICENSE ├── README.md ├── dynamic_inventory_cm ├── dynamic_inventory_cm_py2 ├── dynamic_inventory_cm_py3 ├── group_vars ├── all ├── cdh_servers.yml └── db_server.yml ├── hosts ├── icla ├── Cloudera_CCLA_25APR2018.pdf └── Cloudera_ICLA_25APR2018.pdf ├── images └── figure_1_ansible_inventory-v2.png ├── realm_join.yaml ├── realm_leave.yaml ├── roles ├── cdh │ ├── tasks │ │ └── main.yml │ └── templates │ │ ├── base.j2 │ │ ├── hbase.j2 │ │ ├── hdfs.j2 │ │ ├── hive.j2 │ │ ├── host.j2 │ │ ├── hue.j2 │ │ ├── impala.j2 │ │ ├── instantiator.j2 │ │ ├── oozie.j2 │ │ ├── sentry.j2 │ │ ├── spark.j2 │ │ ├── spark2.j2 │ │ ├── yarn.j2 │ │ └── zookeeper.j2 ├── cm_agents │ └── tasks │ │ ├── 36322.yml │ │ └── main.yml ├── cm_repo │ └── tasks │ │ └── main.yml ├── epel │ └── tasks │ │ └── main.yml ├── java │ └── tasks │ │ ├── install_jce_from_config.yml │ │ ├── install_jce_from_zip.yml │ │ └── main.yml ├── krb5 │ ├── client │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yaml │ ├── common │ │ └── defaults │ │ │ └── main.yml │ └── server │ │ ├── meta │ │ └── main.yml │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ ├── kadm5.acl.j2 │ │ └── kdc.conf.j2 ├── mariadb │ ├── handlers │ │ └── main.yml │ ├── tasks │ │ ├── databases.yml │ │ ├── main.yml │ │ └── mysql_secure_installation.yml │ └── templates │ │ └── my.cnf.j2 ├── mysql_connector │ └── tasks │ │ └── main.yml ├── realm │ ├── join │ │ ├── tasks │ │ │ └── main.yaml │ │ └── templates │ │ │ ├── nscd.conf.j2 │ │ │ ├── realmd.conf.j2 │ │ │ └── sssd.conf.j2 │ └── leave │ │ └── tasks │ │ └── main.yaml ├── rngd │ ├── handlers │ │ └── main.yaml │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── rngd.service.j2 └── scm │ ├── handlers │ └── main.yml │ ├── meta │ └── main.yml │ ├── tasks │ ├── cms.yml │ ├── license.yml │ ├── main.yml │ └── scm.yml │ └── templates │ ├── cms_base.j2 │ ├── scm.j2 │ └── scm_host_list.j2 ├── site.yml └── templates └── krb5.conf.j2 /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.so 3 | *.iml 4 | .idea/ 5 | .DS_Store 6 | docker 7 | *test* 8 | *.retry 9 | .vagrant 10 | Vagrantfile 11 | venv -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | © 2016 Cloudera, Inc. All rights reserved. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloudera Playbook 2 | 3 | An Ansible Playbook that installs the Cloudera stack on RHEL/CentOS 4 | 5 | ## Running the playbook 6 | 7 | * Setup an [Ansible Control Machine](http://docs.ansible.com/ansible/intro_installation.html). 8 | 9 | **Please do not use Ansible 2.9.0**. This version has an [issue with templating](https://github.com/ansible/ansible/issues/64745) which causes the playbook execution to fail. Instead, use any 2.8.x version or a later 2.9.x version as these are not affected. 10 | 11 | * Create Ansible configuration (optional): 12 | 13 | ```ini 14 | $ vi ~/.ansible.cfg 15 | 16 | [defaults] 17 | # disable key check if host is not initially in 'known_hosts' 18 | host_key_checking = False 19 | 20 | [ssh_connection] 21 | # if True, make ansible use scp if the connection type is ssh (default is sftp) 22 | scp_if_ssh = True 23 | ``` 24 | 25 | * Create [Inventory](http://docs.ansible.com/ansible/intro_inventory.html) of cluster hosts: 26 | 27 | ```ini 28 | $ vi ~/ansible_hosts 29 | 30 | [scm_server] 31 | host1.example.com 32 | 33 | [db_server] 34 | host2.example.com 35 | 36 | [krb5_server] 37 | host3.example.com 38 | 39 | [utility_servers:children] 40 | scm_server 41 | db_server 42 | krb5_server 43 | 44 | [edge_servers] 45 | host4.example.com host_template=HostTemplate-Edge role_ref_names=HDFS-HTTPFS-1 46 | 47 | [master_servers] 48 | host5.example.com host_template=HostTemplate-Master1 49 | host6.example.com host_template=HostTemplate-Master2 50 | host7.example.com host_template=HostTemplate-Master3 51 | 52 | [worker_servers] 53 | host8.example.com 54 | host9.example.com 55 | host10.example.com 56 | 57 | [worker_servers:vars] 58 | host_template=HostTemplate-Workers 59 | 60 | [cdh_servers:children] 61 | utility_servers 62 | edge_servers 63 | master_servers 64 | worker_servers 65 | ``` 66 | 67 | **Important**: fully qualified domain name (FQDN) is mandatory in the ansible_hosts file 68 | 69 | * Run playbook 70 | 71 | ```shell 72 | $ ansible-playbook -i ~/ansible_hosts cloudera-playbook/site.yml 73 | 74 | -i INVENTORY 75 | inventory host path or comma separated host list (default=/etc/ansible/hosts) 76 | ``` 77 | 78 | Ansible communicates with the hosts defined in the inventory over SSH. It assumes you’re using SSH keys to authenticate so your public SSH key should exist in ``authorized_keys`` on those hosts. Your user will need sudo privileges to install the required packages. 79 | 80 | By default Ansible will connect to the remote hosts using the current user (as SSH would). To override the remote user name you can specify the ``--user`` option in the command, or add the following variables to the inventory: 81 | 82 | ```ini 83 | [all:vars] 84 | ansible_user=ec2-user 85 | ``` 86 | 87 | AWS users can use Ansible’s ``--private-key`` option to authenticate using a PEM file instead of SSH keys. 88 | 89 | ## Enabling Kerberos 90 | 91 | The playbook can install a local MIT KDC and configure Hadoop Security. To enable Hadoop Security: 92 | 93 | * Specify the '[krb5_server]' host in the inventory (see above) 94 | * Set 'krb5_kdc_type' to 'MIT KDC' in ``group_vars/krb5_server.yml`` 95 | 96 | ## Overriding CDH service/role configuration 97 | 98 | The playbook uses [Cloudera Manager Templates](https://www.cloudera.com/documentation/enterprise/latest/topics/install_cluster_template.html) to provision a cluster. 99 | As part of the template import process Cloudera Manager applies [Autoconfiguration](https://www.cloudera.com/documentation/enterprise/latest/topics/cm_mc_autoconfig.html) 100 | rules that set properties such as memory and CPU allocations for various roles. 101 | 102 | If the cluster has different hardware or operational requirements then you can override these properties in ``group_vars/cdh_servers``. 103 | For example: 104 | 105 | ``` 106 | cdh_services: 107 | - type: hdfs 108 | datanode_java_heapsize: 10737418240 109 | ``` 110 | 111 | These properties get added as variables to the rendered template's instantiator block and can be referenced from the service configs. 112 | For example ``roles/cdh/templates/hdfs.j2``: 113 | 114 | ```json 115 | "roleType": "DATANODE", 116 | "configs": [{ 117 | "name": "datanode_java_heapsize", 118 | "variable": "DATANODE_JAVA_HEAPSIZE" 119 | } 120 | ``` 121 | 122 | ## Dynamic Inventory Script for Cloudera Manager 123 | 124 | To make integration easier, Gabor Roczei created a dynamic inventory [script](https://github.com/cloudera/cloudera-playbook/blob/master/dynamic_inventory_cm_py2) that allows Ansible to gather data from Cloudera Manager. Its main advantages are: 125 | 126 | * Cache management of inventory for better performance 127 | * Cloudera Manager’s HTTP cookie handling 128 | * Support for multiple Cloudera Manager instances 129 | * SSL-friendly, as the root CA check of the Cloudera Manager server can be disabled or enabled 130 | 131 |

132 | High level architecture of Ansible dynamic inventory vs. Cloudera Managers
133 | High level architecture of Ansible dynamic inventory vs. Cloudera Managers 134 |

135 | 136 | ### Configuration 137 | 138 | **Step 1**: Configuration of the related Cloudera Manager(s) 139 | 140 | ```ini 141 | $ export CM_URL=https://cm1.example.com:7183,https://cm2.example.com:7183 142 | $ export CM_USERNAME=username 143 | ``` 144 | 145 | Other optional configuration parameters: 146 | 147 | ```ini 148 | $ export CM_CACHE_TIME_SEC=3600 149 | $ export CM_DISABLE_CA_CHECK=True 150 | $ export CM_TIMEOUT_SEC=60 151 | $ export CM_DEBUG=False 152 | ``` 153 | 154 | Note: We recommend adding these environment variables to the startup file of your shell. For example: $HOME/.bashrc 155 | 156 | **Step 2**: Installation of the git package: 157 | 158 | ``` 159 | # yum install git 160 | ``` 161 | 162 | **Step 3**: Installation of the Ansible package: 163 | 164 | ``` 165 | # yum install ansible 166 | ``` 167 | 168 | **Step 4**: Clone the cloudera-playbook git repository: 169 | 170 | ``` 171 | $ git clone https://github.com/cloudera/cloudera-playbook 172 | ``` 173 | 174 | Note: The cloudera-playbook git repository is not officially supported by Cloudera, but the authors recommend using it. 175 | 176 | **Step 5**: Setup the default Ansible inventory and other useful Ansible [parameters](https://raw.githubusercontent.com/ansible/ansible/devel/examples/ansible.cfg): 177 | 178 | ```ini 179 | $ vi $HOME/.ansible.cfg 180 | [defaults] 181 | # Python 2 version: 182 | inventory = $HOME/cloudera-playbook/dynamic_inventory_cm_py2 183 | # Python 3 version: 184 | # inventory = $HOME/cloudera-playbook/dynamic_inventory_cm_py3 185 | # Do not gather the host information (facts) by default. This can give significant speedups for large clusters. 186 | # gathering = explicit 187 | # Disable key check if host is not initially in 'known_hosts' 188 | host_key_checking = False 189 | [ssh_connection] 190 | # If it is True, make ansible use scp if the connection type is ssh (default is sftp) 191 | scp_if_ssh = True 192 | ``` 193 | 194 | Note: Update the inventory path of the dynamic_inventory_cm_py2 if necessary 195 | 196 | **Step 6**: Change the working directory to cloudera-playbook 197 | 198 | ``` 199 | $ cd cloudera-playbook 200 | ``` 201 | 202 | **Step 7**: The available Cloudera Manager clusters (Ansible groups, such as Cluster_1, Balaton) can be listed with the following command: 203 | 204 | ``` 205 | $ ./dynamic_inventory_cm_py2 --list 206 | ``` 207 | 208 | Note: The cache of the Cloudera Manager inventory can be refreshed with the following command: 209 | 210 | ``` 211 | $ ./dynamic_inventory_cm_py2 --refresh-cache 212 | ``` 213 | 214 | **Step 8**: Setup the SSH public key authentication for remote hosts: 215 | 216 | The big advantage of this is that with ad-hoc commands, you do not need to enter your password each time you run the command, but only the first time you enter the private key password. 217 | 218 | If the ~/.ssh/id_rsa.pub and ~/.ssh/id_rsa files do not exist, they need to be generated with the ssh-keygen command prior to connecting to the managed hosts. 219 | 220 | Launch a subshell with the following command: 221 | 222 | ``` 223 | $ ssh-agent bash 224 | ``` 225 | 226 | You will execute the rest of the commands in this How-To article in the subshell. 227 | 228 | Add the SSH private key into the SSH authentication agent: 229 | 230 | ``` 231 | $ ssh-add ~/.ssh/id_rsa 232 | ``` 233 | 234 | Validate: 235 | 236 | ``` 237 | $ ssh-add -L 238 | ``` 239 | 240 | Upload the SSH public key (id_rsa.pub) to the managed hosts: 241 | 242 | ``` 243 | $ ansible all -m authorized_key -a key="{{ lookup('file', '~/.ssh/id_rsa.pub') }} user=$USER" --ask-pass -u $USER --become-user $USER 244 | ``` 245 | 246 | For example, you can use the root user: 247 | 248 | ``` 249 | $ ansible all -m authorized_key -a key="{{ lookup('file', '~/.ssh/id_rsa.pub') }} user=root" --ask-pass -u root 250 | ``` 251 | 252 | Note: If you do not want to use SSH public key authentication, add the --ask-pass parameter each time you run the Ansible command. 253 | 254 | 255 | **Step 9**: Test remote host connectivity (optional): 256 | 257 | ``` 258 | $ ansible all -m ping -u $USER --become-user $USER 259 | ``` 260 | 261 | For example, you can execute the command with the root user: 262 | 263 | ``` 264 | $ ansible all -m ping -u root 265 | ``` 266 | 267 | **Step 10**: The ad-hoc command feature enables running single and arbitrary Linux commands on all hosts. You can use this to troubleshoot slow group resolution issues. 268 | 269 | The following commands are example ad-hoc commands where Balaton is a group of hosts that is a cluster in Cloudera Manager: 270 | 271 | ``` 272 | $ ansible Balaton -u $USER --become-user $USER -m command -o -a "time id -Gn $USER" 273 | $ ansible all -u $USER --become-user $USER -m command -o -a "date" 274 | ``` 275 | 276 | The following example uses the root user: 277 | 278 | ``` 279 | $ ansible Balaton -m command -o -a "time id -Gn testuser" -u root 280 | $ ansible all -m command -o -a "date" -u root 281 | ``` 282 | 283 | Further information about dynamic inventory and ad-hoc commands can be found in the Ansible documentation: 284 | 285 | * [Developing Dynamic Inventory](http://docs.ansible.com/ansible/latest/dev_guide/developing_inventory.html) 286 | * [Documentation of Ansible Ad-Hoc commands](http://docs.ansible.com/ansible/latest/intro_adhoc.html) 287 | 288 | ### SSSD setup with Ansible (applicable for RHEL 7 / CentOS 7) 289 | 290 | Cloudera blog articles: 291 | 292 | * [Best Practices Guide for Systems Security Services Daemon Configuration and Installation – Part 1](https://blog.cloudera.com/best-practices-guide-for-systems-security-services-daemon-configuration-and-installation-part-1) 293 | * [How-to: Automate the Systems Security Services Daemon Installation and Troubleshoot it with Ansible – Part 2](https://blog.cloudera.com/how-to-automate-the-systems-security-services-daemon-installation-and-troubleshoot-it-with-ansible-part-2) 294 | 295 | **Step 1**: Edit the default variables in group_vars/all: 296 | 297 | ``` 298 | krb5_realm: AD.SEC.EXAMPLE.COM 299 | krb5_kdc_type: Active Directory 300 | krb5_kdc_host: w2k8-1.ad.sec.example.com 301 | ad_domain: "{{ krb5_realm.lower() }}" 302 | computer_ou: ou=computer_hosts,ou=hadoop_prd,dc=ad,dc=sec,dc=example,dc=com 303 | ldap_group_search_base: OU=groups,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com 304 | ldap_user_search_base: DC=ad,DC=sec,DC=example,DC=com?subtree?(memberOf=CN=hadoop_users,OU=groups,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com) 305 | override_gid: 999999 306 | ad_site: Default-First-Site-Name 307 | ``` 308 | 309 | **Step 2**: Enable kerberos on the hosts: 310 | 311 | If necessary, update this template file (See the Ansible [Templating (Jinja2)](https://docs.ansible.com/ansible/latest/user_guide/playbooks_templating.html) documentation for more information): 312 | 313 | ``` 314 | templates/krb5.conf.j2 315 | ``` 316 | 317 | Run this command to apply it on the managed hosts: 318 | 319 | ``` 320 | $ ansible-playbook --tags krb5_client -u root site.yml 321 | ``` 322 | 323 | **Step 3**: Join the host(s) to realm: 324 | 325 | If necessary, update these template files (See the Ansible [Templating (Jinja2)](https://docs.ansible.com/ansible/latest/user_guide/playbooks_templating.html) documentation for more information): 326 | 327 | ``` 328 | roles/realm/join/templates/sssd.conf.j2 329 | roles/realm/join/templates/realmd.conf.j2 330 | roles/realm/join/templates/nscd.conf.j2 331 | ``` 332 | 333 | Run this command to apply it on all managed hosts: 334 | 335 | ``` 336 | $ ansible-playbook -u root realm_join.yaml 337 | bind user: administrator 338 | bind password: 339 | ``` 340 | 341 | Run this command to apply it on a cluster (for example: Balaton) (See the Ansible [Best Practices](https://docs.ansible.com/ansible/latest/user_guide/playbooks_best_practices.html) documentation for more information): 342 | 343 | ``` 344 | $ ansible-playbook --limit Balaton -u root realm_join.yaml 345 | bind user: administrator 346 | bind password: 347 | ``` 348 | 349 | Remove all hosts from the realm with this command: 350 | 351 | ``` 352 | $ ansible-playbook -u root realm_leave.yaml 353 | ``` 354 | 355 | Remove the Balaton hosts from the realm with this command (See the Ansible [Best Practices](https://docs.ansible.com/ansible/latest/user_guide/playbooks_best_practices.html) documentation for more information): 356 | 357 | ``` 358 | $ ansible-playbook --limit Balaton -u root realm_leave.yaml 359 | ``` 360 | 361 | ## How do I contribute code? 362 | You need to first sign and return an 363 | [ICLA](https://github.com/cloudera/cloudera-playbook/blob/master/icla/Cloudera_ICLA_25APR2018.pdf) 364 | and 365 | [CCLA](https://github.com/cloudera/cloudera-playbook/blob/master/icla/Cloudera_CCLA_25APR2018.pdf) 366 | before we can accept and redistribute your contribution. Once these are submitted you are 367 | free to start contributing to cloudera-playbook. Submit these to CLA@cloudera.com. 368 | 369 | ### Main steps 370 | * Fork the repo and create a topic branch 371 | * Push commits to your repo 372 | * Create a pull request! 373 | 374 | ### Find 375 | We use Github issues to track bugs for this project. Find an issue that you would like to 376 | work on (or file one if you have discovered a new issue!). If no-one is working on it, 377 | assign it to yourself only if you intend to work on it shortly. 378 | 379 | ### Fix 380 | 381 | Please write a good, clear commit message, with a short, descriptive title and 382 | a message that is exactly long enough to explain what the problem was, and how it was 383 | fixed. 384 | 385 | License 386 | ----------- 387 | [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) 388 | -------------------------------------------------------------------------------- /dynamic_inventory_cm: -------------------------------------------------------------------------------- 1 | dynamic_inventory_cm_py2 -------------------------------------------------------------------------------- /dynamic_inventory_cm_py2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Created by Gabor Roczei 6 | 7 | ============================================================================= 8 | Ansible Dynamic Inventory Script for Cloudera Manager 9 | ============================================================================= 10 | 11 | The available Cloudera Manager clusters (Ansible groups, for example: Cluster_1, Balaton) 12 | can be listed with the following command: 13 | 14 | dynamic_inventory_cm --list 15 | 16 | Example Ansible ad-hoc commands: 17 | 18 | ansible all -u $USER --become-user $USER -i dynamic_inventory_cm --list-hosts 19 | ansible Balaton --ask-pass -u $USER --become-user $USER -i dynamic_inventory_cm -o -m command -a "time id -Gn $USER" 20 | ansible Cluster_1 --ask-pass -u $USER --become-user $USER -i dynamic_inventory_cm -o -m command -a "date" 21 | ansible --ask-pass -u $USER --become-user $USER -i dynamic_inventory_cm -o -m ping all 22 | ''' 23 | 24 | import os 25 | import time 26 | import sys 27 | import argparse 28 | import json 29 | import urllib2 30 | import ssl 31 | import base64 32 | import getpass 33 | import cookielib 34 | from urlparse import urlparse 35 | 36 | # Cache time (seconds) 37 | default_cache_time_sec = 30 * 24 * 60 * 60 38 | 39 | # Disabling SSL CA check 40 | default_disable_ca_check = False 41 | 42 | # Cloudera Manager API timeout (seconds) 43 | default_timeout_sec = 60 44 | 45 | # Debug mode 46 | default_debug = False 47 | 48 | def save_cookie(cookiejar, filename): 49 | lwp_cookiejar = cookielib.LWPCookieJar() 50 | for cookie in cookiejar: 51 | parameters = dict(vars(cookie).items()) 52 | parameters["rest"] = parameters["_rest"] 53 | del parameters["_rest"] 54 | cookie = cookielib.Cookie(**parameters) 55 | lwp_cookiejar.set_cookie(cookie) 56 | lwp_cookiejar.save(filename, ignore_discard = True) 57 | os.chmod(filename, 0600) 58 | 59 | def load_cookie(filename): 60 | lwp_cookiejar = cookielib.LWPCookieJar() 61 | lwp_cookiejar.load(filename, ignore_discard = True) 62 | return lwp_cookiejar 63 | 64 | class ClouderaManagerInventory: 65 | 66 | def __init__(self): 67 | self.inventory = {"_meta" : { "hostvars": {} }} 68 | self.cookie_path = os.environ["HOME"] + os.sep + ".cm" + os.sep + ".cm_cookie" 69 | self.json_path = os.environ["HOME"] + os.sep + ".cm" + os.sep + ".cm_inventory.json" 70 | self.api_version = "v30" 71 | 72 | cache_time_sec = os.getenv("CM_CACHE_TIME_SEC") 73 | disable_ca_check = os.getenv("CM_DISABLE_CA_CHECK") 74 | timeout_sec = os.getenv("CM_TIMEOUT_SEC") 75 | debug = os.getenv("CM_DEBUG") 76 | 77 | if not os.path.exists(os.environ["HOME"] + os.sep + ".cm"): 78 | os.makedirs(os.environ["HOME"] + os.sep + ".cm") 79 | os.chmod(os.environ["HOME"] + os.sep + ".cm", 0700) 80 | 81 | if cache_time_sec == None: 82 | self.cache_time_sec = default_cache_time_sec 83 | else: 84 | try: 85 | self.cache_time_sec = int(cache_time_sec) 86 | except Exception, e: 87 | if "invalid literal for int" in str(e): 88 | sys.stderr.write("The value is incorrect in CM_CACHE_TIME_SEC: " + cache_time_sec + ". It is not a valid integer number.\n") 89 | sys.exit(-1) 90 | else: 91 | sys.stderr.write(str(e)) 92 | sys.exit(-1) 93 | 94 | if timeout_sec == None: 95 | self.timeout_sec = default_timeout_sec 96 | else: 97 | try: 98 | self.timeout_sec = int(timeout_sec) 99 | except Exception, e: 100 | if "invalid literal for int" in str(e): 101 | sys.stderr.write("The value is incorrect in CM_TIMEOUT_SEC: " + timeout_sec + ". It is not a valid integer number.\n") 102 | sys.exit(-1) 103 | else: 104 | sys.stderr.write(str(e)) 105 | sys.exit(-1) 106 | 107 | if disable_ca_check == None: 108 | self.disable_ca_check = default_disable_ca_check 109 | elif disable_ca_check.lower() == "true": 110 | self.disable_ca_check = True 111 | elif disable_ca_check.lower() == "false": 112 | self.disable_ca_check = False 113 | else: 114 | sys.stderr.write("Unknown value in CM_DISABLE_CA_CHECK: " + disable_ca_check + ". Valid values can only be True or False\n") 115 | sys.exit(-1) 116 | 117 | if debug == None: 118 | self.debug = default_debug 119 | elif debug.lower() == "true": 120 | self.debug = True 121 | elif debug.lower() == "false": 122 | self.debug = False 123 | else: 124 | sys.stderr.write("Unknown value in CM_DEBUG: " + debug + ". Valid values can only be True or False\n") 125 | sys.exit(-1) 126 | 127 | self.read_args() 128 | 129 | def cm_connection(self, url, username, password, cookie): 130 | try: 131 | if self.disable_ca_check: 132 | context = ssl._create_unverified_context() 133 | else: 134 | context = None 135 | 136 | cookie_handler = urllib2.HTTPCookieProcessor(cookie) 137 | 138 | password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() 139 | password_manager.add_password(realm = "Cloudera Manager", 140 | uri = url + "/api/v1", 141 | user = username, 142 | passwd = password) 143 | 144 | auth_handler = urllib2.HTTPBasicAuthHandler(password_manager) 145 | 146 | if self.debug: 147 | debuglevel = 1 148 | else: 149 | debuglevel = 0 150 | 151 | http_handler = urllib2.HTTPHandler(debuglevel = debuglevel) 152 | https_handler = urllib2.HTTPSHandler(context = context, debuglevel = debuglevel) 153 | error_handler = urllib2.HTTPErrorProcessor() 154 | 155 | opener = urllib2.build_opener(cookie_handler, auth_handler, http_handler, https_handler, error_handler) 156 | urllib2.install_opener(opener) 157 | 158 | auth_string = base64.encodestring("%s:%s" % (username, password)).replace("\n", "") 159 | headers = {"Authorization" : "Basic %s" % auth_string} 160 | 161 | version_url = url + "/api/version" 162 | version_request = urllib2.Request(version_url, None, headers) 163 | version_response = urllib2.urlopen(version_request, timeout = self.timeout_sec).read() 164 | 165 | #if version_response: 166 | # self.api_version = version_response 167 | 168 | hosts_url = url + "/api/" + self.api_version + "/hosts" 169 | hosts_response = opener.open(hosts_url, timeout = self.timeout_sec).read() 170 | host_names = {} 171 | 172 | if hosts_response: 173 | hosts = json.loads(hosts_response) 174 | for host in hosts["items"]: 175 | hostid = host.get("hostId") 176 | host_names[hostid] = host.get("hostname") 177 | 178 | clusters_url = url + "/api/" + self.api_version + "/clusters" 179 | clusters_response = opener.open(clusters_url, timeout = self.timeout_sec).read() 180 | 181 | if clusters_response: 182 | clusters = json.loads(clusters_response) 183 | for cluster in clusters["items"]: 184 | cluster_name = cluster.get("displayName") 185 | cluster_name_updated = cluster_name.replace(" ", "_") 186 | cluster_name_updated = cluster_name.replace("-", "_") 187 | hosts = {"hosts" : []} 188 | i = 2 189 | while self.inventory.has_key(cluster_name_updated): 190 | if self.inventory.has_key(cluster_name_updated + "_" + str(i)): 191 | i = i + 1 192 | else: 193 | cluster_name_updated = cluster_name_updated + "_" + str(i) 194 | break 195 | self.inventory[cluster_name_updated] = hosts 196 | clusters_hosts_url = url + "/api/" + self.api_version + "/clusters/" + cluster_name + "/hosts" 197 | clusters_hosts_url = urllib2.quote(clusters_hosts_url,":/") 198 | clusters_hosts_response = opener.open(clusters_hosts_url, timeout = self.timeout_sec).read() 199 | 200 | if clusters_hosts_response: 201 | hostids = json.loads(clusters_hosts_response) 202 | for hostid in hostids["items"]: 203 | hostid = hostid.get("hostId") 204 | host = host_names[hostid] 205 | self.inventory[cluster_name_updated]["hosts"].append(host) 206 | 207 | parsed_url = urlparse(url) 208 | save_cookie(cookie, self.cookie_path + "." + str(parsed_url.hostname)) 209 | 210 | except Exception, e: 211 | if "Bad credentials" in str(e): 212 | if os.path.exists(self.cookie_path): 213 | os.remove(self.cookie_path) 214 | return 1 215 | elif "CERTIFICATE_VERIFY_FAILED" in str(e): 216 | sys.stderr.write("URL: " + url + "\n") 217 | sys.stderr.write(str(e) + "\n") 218 | sys.stderr.write("\nThe root CA check for Cloudera Manager server(s) can be disabled or enabled with this:\n\n") 219 | sys.stderr.write("export CM_DISABLE_CA_CHECK=True\n\n") 220 | sys.exit(-1) 221 | else: 222 | sys.stderr.write("URL: " + url + "\n") 223 | sys.stderr.write(str(e) + "\n") 224 | sys.exit(-1) 225 | 226 | def read_args(self): 227 | parser = argparse.ArgumentParser() 228 | parser.add_argument("--list", help = "List the clusters and the hosts", action = "store_true") 229 | parser.add_argument("--host", help = "", action = "store") 230 | parser.add_argument("--refresh-cache", help = "Refresh the (Cloudera Manager) inventory cache" , action = "store_true") 231 | self.args = parser.parse_args() 232 | 233 | if self.args.refresh_cache: 234 | self.option_refresh_cache() 235 | 236 | if self.args.list: 237 | self.option_list() 238 | 239 | if self.args.host: 240 | self.option_host() 241 | 242 | def option_list(self): 243 | if os.path.exists(self.json_path) and (time.time() - os.path.getmtime(self.json_path) < self.cache_time_sec): 244 | json_data = open(self.json_path, "r") 245 | self.inventory = json.load(json_data) 246 | print json.dumps(self.inventory, sort_keys = True, indent = 4, separators=(",", ": ")) 247 | sys.exit(0) 248 | 249 | self.option_refresh_cache() 250 | print json.dumps(self.inventory, sort_keys = True, indent = 4, separators=(",", ": ")) 251 | 252 | def option_refresh_cache(self): 253 | cm_urls = os.getenv("CM_URL") 254 | username = os.getenv("CM_USERNAME") 255 | password = "" 256 | 257 | if cm_urls == None: 258 | sys.stderr.write("CM_URL shell variable has not been set, configure the Cloudera Manager's URL. Examples:\n\n") 259 | sys.stderr.write("export CM_URL=https://host-10-17-101-73.coe.example.com:7183\n") 260 | sys.stderr.write("export CM_URL=https://host-10-17-101-73.coe.example.com:7183,http://nightly512-1.vpc.example.com:7180\n\n") 261 | sys.exit(-1) 262 | 263 | if username == None: 264 | sys.stderr.write("CM_USERNAME shell variable has not been set, configure the Cloudera Manager's username. Example:\n\n") 265 | sys.stderr.write("export CM_USERNAME=username\n\n") 266 | sys.exit(-1) 267 | 268 | urls = cm_urls.split(",") 269 | 270 | for url in urls: 271 | url = url.strip() 272 | parsed_url = urlparse(url) 273 | cookie_path = self.cookie_path + "." + str(parsed_url.hostname) 274 | if os.path.isfile(cookie_path): 275 | cookie = load_cookie(cookie_path) 276 | result = self.cm_connection(url, username, password, cookie) 277 | if result != 1: 278 | continue 279 | 280 | cookie = cookielib.LWPCookieJar() 281 | while True: 282 | prompt_message = "Cloudera Manager's URL: " + url + "\nCloudera Manager's password: " 283 | password = getpass.getpass(prompt_message) 284 | result = self.cm_connection(url, username, password, cookie) 285 | if result != 1: 286 | break 287 | user_input = raw_input("Cloudera Manager's username (default: %s): " % username) 288 | if user_input: 289 | username = user_input.strip() 290 | 291 | with open(self.json_path, "w") as outfile: 292 | json.dump(self.inventory, outfile) 293 | 294 | def option_host(self): 295 | print json.dumps(self.inventory, sort_keys = True, indent = 4, separators=(",", ": ")) 296 | 297 | if __name__ == "__main__": 298 | ClouderaManagerInventory() 299 | -------------------------------------------------------------------------------- /dynamic_inventory_cm_py3: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Created by Gabor Roczei 6 | 7 | ============================================================================= 8 | Ansible Dynamic Inventory Script for Cloudera Manager 9 | ============================================================================= 10 | 11 | The available Cloudera Manager clusters (Ansible groups, for example: Cluster_1, Balaton) 12 | can be listed with the following command: 13 | 14 | dynamic_inventory_cm --list 15 | 16 | Example Ansible ad-hoc commands: 17 | 18 | ansible all -u $USER --become-user $USER -i dynamic_inventory_cm --list-hosts 19 | ansible Balaton --ask-pass -u $USER --become-user $USER -i dynamic_inventory_cm -o -m command -a "time id -Gn $USER" 20 | ansible Cluster_1 --ask-pass -u $USER --become-user $USER -i dynamic_inventory_cm -o -m command -a "date" 21 | ansible --ask-pass -u $USER --become-user $USER -i dynamic_inventory_cm -o -m ping all 22 | ''' 23 | 24 | import os 25 | import time 26 | import sys 27 | import argparse 28 | import json 29 | import urllib.request, urllib.error, urllib.parse 30 | import ssl 31 | import base64 32 | import getpass 33 | import http.cookiejar 34 | from urllib.parse import urlparse 35 | 36 | # Cache time (seconds) 37 | default_cache_time_sec = 30 * 24 * 60 * 60 38 | 39 | # Disabling SSL CA check 40 | default_disable_ca_check = False 41 | 42 | # Cloudera Manager API timeout (seconds) 43 | default_timeout_sec = 60 44 | 45 | # Debug mode 46 | default_debug = False 47 | 48 | def save_cookie(cookiejar, filename): 49 | lwp_cookiejar = http.cookiejar.LWPCookieJar() 50 | for cookie in cookiejar: 51 | parameters = dict(list(vars(cookie).items())) 52 | parameters["rest"] = parameters["_rest"] 53 | del parameters["_rest"] 54 | cookie = http.cookiejar.Cookie(**parameters) 55 | lwp_cookiejar.set_cookie(cookie) 56 | lwp_cookiejar.save(filename, ignore_discard = True) 57 | os.chmod(filename, 0o600) 58 | 59 | def load_cookie(filename): 60 | lwp_cookiejar = http.cookiejar.LWPCookieJar() 61 | lwp_cookiejar.load(filename, ignore_discard = True) 62 | return lwp_cookiejar 63 | 64 | class ClouderaManagerInventory: 65 | 66 | def __init__(self): 67 | self.inventory = {"_meta" : { "hostvars": {} }} 68 | self.cookie_path = os.environ["HOME"] + os.sep + ".cm" + os.sep + ".cm_cookie" 69 | self.json_path = os.environ["HOME"] + os.sep + ".cm" + os.sep + ".cm_inventory.json" 70 | self.api_version = "v30" 71 | 72 | cache_time_sec = os.getenv("CM_CACHE_TIME_SEC") 73 | disable_ca_check = os.getenv("CM_DISABLE_CA_CHECK") 74 | timeout_sec = os.getenv("CM_TIMEOUT_SEC") 75 | debug = os.getenv("CM_DEBUG") 76 | 77 | if not os.path.exists(os.environ["HOME"] + os.sep + ".cm"): 78 | os.makedirs(os.environ["HOME"] + os.sep + ".cm") 79 | os.chmod(os.environ["HOME"] + os.sep + ".cm", 0o700) 80 | 81 | if cache_time_sec == None: 82 | self.cache_time_sec = default_cache_time_sec 83 | else: 84 | try: 85 | self.cache_time_sec = int(cache_time_sec) 86 | except Exception as e: 87 | if "invalid literal for int" in str(e): 88 | sys.stderr.write("The value is incorrect in CM_CACHE_TIME_SEC: " + cache_time_sec + ". It is not a valid integer number.\n") 89 | sys.exit(-1) 90 | else: 91 | sys.stderr.write(str(e)) 92 | sys.exit(-1) 93 | 94 | if timeout_sec == None: 95 | self.timeout_sec = default_timeout_sec 96 | else: 97 | try: 98 | self.timeout_sec = int(timeout_sec) 99 | except Exception as e: 100 | if "invalid literal for int" in str(e): 101 | sys.stderr.write("The value is incorrect in CM_TIMEOUT_SEC: " + timeout_sec + ". It is not a valid integer number.\n") 102 | sys.exit(-1) 103 | else: 104 | sys.stderr.write(str(e)) 105 | sys.exit(-1) 106 | 107 | if disable_ca_check == None: 108 | self.disable_ca_check = default_disable_ca_check 109 | elif disable_ca_check.lower() == "true": 110 | self.disable_ca_check = True 111 | elif disable_ca_check.lower() == "false": 112 | self.disable_ca_check = False 113 | else: 114 | sys.stderr.write("Unknown value in CM_DISABLE_CA_CHECK: " + disable_ca_check + ". Valid values can only be True or False\n") 115 | sys.exit(-1) 116 | 117 | if debug == None: 118 | self.debug = default_debug 119 | elif debug.lower() == "true": 120 | self.debug = True 121 | elif debug.lower() == "false": 122 | self.debug = False 123 | else: 124 | sys.stderr.write("Unknown value in CM_DEBUG: " + debug + ". Valid values can only be True or False\n") 125 | sys.exit(-1) 126 | 127 | self.read_args() 128 | 129 | def cm_connection(self, url, username, password, cookie): 130 | try: 131 | if self.disable_ca_check: 132 | context = ssl._create_unverified_context() 133 | else: 134 | context = None 135 | 136 | cookie_handler = urllib.request.HTTPCookieProcessor(cookie) 137 | 138 | password_manager = urllib.request.HTTPPasswordMgrWithDefaultRealm() 139 | password_manager.add_password(realm = "Cloudera Manager", 140 | uri = url + "/api/v1", 141 | user = username, 142 | passwd = password) 143 | 144 | auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) 145 | 146 | if self.debug: 147 | debuglevel = 1 148 | else: 149 | debuglevel = 0 150 | 151 | http_handler = urllib.request.HTTPHandler(debuglevel = debuglevel) 152 | https_handler = urllib.request.HTTPSHandler(context = context, debuglevel = debuglevel) 153 | error_handler = urllib.request.HTTPErrorProcessor() 154 | 155 | opener = urllib.request.build_opener(cookie_handler, auth_handler, http_handler, https_handler, error_handler) 156 | urllib.request.install_opener(opener) 157 | 158 | userpass = username + ":" + password 159 | auth_string = base64.b64encode(userpass.encode()).decode() 160 | headers = {"Authorization" : "Basic %s" % auth_string} 161 | 162 | version_url = url + "/api/version" 163 | version_request = urllib.request.Request(version_url, None, headers) 164 | version_response = urllib.request.urlopen(version_request, timeout = self.timeout_sec).read() 165 | 166 | #if version_response: 167 | # self.api_version = version_response.decode() 168 | 169 | hosts_url = url + "/api/" + self.api_version + "/hosts" 170 | hosts_response = opener.open(hosts_url, timeout = self.timeout_sec).read() 171 | host_names = {} 172 | 173 | if hosts_response: 174 | hosts = json.loads(hosts_response.decode()) 175 | for host in hosts["items"]: 176 | hostid = host.get("hostId") 177 | host_names[hostid] = host.get("hostname") 178 | 179 | clusters_url = url + "/api/" + self.api_version + "/clusters" 180 | clusters_response = opener.open(clusters_url, timeout = self.timeout_sec).read() 181 | 182 | if clusters_response: 183 | clusters = json.loads(clusters_response.decode()) 184 | for cluster in clusters["items"]: 185 | cluster_name = cluster.get("displayName") 186 | cluster_name_updated = cluster_name.replace(" ", "_") 187 | cluster_name_updated = cluster_name.replace("-", "_") 188 | hosts = {"hosts" : []} 189 | i = 2 190 | while cluster_name_updated in self.inventory: 191 | if cluster_name_updated + "_" + str(i) in self.inventory: 192 | i = i + 1 193 | else: 194 | cluster_name_updated = cluster_name_updated + "_" + str(i) 195 | break 196 | self.inventory[cluster_name_updated] = hosts 197 | clusters_hosts_url = url + "/api/" + self.api_version + "/clusters/" + cluster_name + "/hosts" 198 | clusters_hosts_url = urllib.parse.quote(clusters_hosts_url,":/") 199 | clusters_hosts_response = opener.open(clusters_hosts_url, timeout = self.timeout_sec).read() 200 | 201 | if clusters_hosts_response: 202 | hostids = json.loads(clusters_hosts_response.decode()) 203 | for hostid in hostids["items"]: 204 | hostid = hostid.get("hostId") 205 | host = host_names[hostid] 206 | self.inventory[cluster_name_updated]["hosts"].append(host) 207 | 208 | parsed_url = urlparse(url) 209 | save_cookie(cookie, self.cookie_path + "." + str(parsed_url.hostname)) 210 | 211 | except Exception as e: 212 | if "Bad credentials" in str(e): 213 | if os.path.exists(self.cookie_path): 214 | os.remove(self.cookie_path) 215 | return 1 216 | elif "CERTIFICATE_VERIFY_FAILED" in str(e): 217 | sys.stderr.write("URL: " + url + "\n") 218 | sys.stderr.write(str(e) + "\n") 219 | sys.stderr.write("\nThe root CA check for Cloudera Manager server(s) can be disabled or enabled with this:\n\n") 220 | sys.stderr.write("export CM_DISABLE_CA_CHECK=True\n\n") 221 | sys.exit(-1) 222 | else: 223 | sys.stderr.write("URL: " + url + "\n") 224 | sys.stderr.write(str(e) + "\n") 225 | sys.exit(-1) 226 | 227 | def read_args(self): 228 | parser = argparse.ArgumentParser() 229 | parser.add_argument("--list", help = "List the clusters and the hosts", action = "store_true") 230 | parser.add_argument("--host", help = "", action = "store") 231 | parser.add_argument("--refresh-cache", help = "Refresh the (Cloudera Manager) inventory cache" , action = "store_true") 232 | self.args = parser.parse_args() 233 | 234 | if self.args.refresh_cache: 235 | self.option_refresh_cache() 236 | 237 | if self.args.list: 238 | self.option_list() 239 | 240 | if self.args.host: 241 | self.option_host() 242 | 243 | def option_list(self): 244 | if os.path.exists(self.json_path) and (time.time() - os.path.getmtime(self.json_path) < self.cache_time_sec): 245 | json_data = open(self.json_path, "r") 246 | self.inventory = json.load(json_data) 247 | print(json.dumps(self.inventory, sort_keys = True, indent = 4, separators=(",", ": "))) 248 | sys.exit(0) 249 | 250 | self.option_refresh_cache() 251 | print(json.dumps(self.inventory, sort_keys = True, indent = 4, separators=(",", ": "))) 252 | 253 | def option_refresh_cache(self): 254 | cm_urls = os.getenv("CM_URL") 255 | username = os.getenv("CM_USERNAME") 256 | password = "" 257 | 258 | if cm_urls == None: 259 | sys.stderr.write("CM_URL shell variable has not been set, configure the Cloudera Manager's URL. Examples:\n\n") 260 | sys.stderr.write("export CM_URL=https://host-10-17-101-73.coe.example.com:7183\n") 261 | sys.stderr.write("export CM_URL=https://host-10-17-101-73.coe.example.com:7183,http://nightly512-1.vpc.example.com:7180\n\n") 262 | sys.exit(-1) 263 | 264 | if username == None: 265 | sys.stderr.write("CM_USERNAME shell variable has not been set, configure the Cloudera Manager's username. Example:\n\n") 266 | sys.stderr.write("export CM_USERNAME=username\n\n") 267 | sys.exit(-1) 268 | 269 | urls = cm_urls.split(",") 270 | 271 | for url in urls: 272 | url = url.strip() 273 | parsed_url = urlparse(url) 274 | cookie_path = self.cookie_path + "." + str(parsed_url.hostname) 275 | if os.path.isfile(cookie_path): 276 | cookie = load_cookie(cookie_path) 277 | result = self.cm_connection(url, username, password, cookie) 278 | if result != 1: 279 | continue 280 | 281 | cookie = http.cookiejar.LWPCookieJar() 282 | while True: 283 | prompt_message = "Cloudera Manager's URL: " + url + "\nCloudera Manager's password: " 284 | password = getpass.getpass(prompt_message) 285 | result = self.cm_connection(url, username, password, cookie) 286 | if result != 1: 287 | break 288 | user_input = input("Cloudera Manager's username (default: %s): " % username) 289 | if user_input: 290 | username = user_input.strip() 291 | 292 | with open(self.json_path, "w") as outfile: 293 | json.dump(self.inventory, outfile) 294 | 295 | def option_host(self): 296 | print(json.dumps(self.inventory, sort_keys = True, indent = 4, separators=(",", ": "))) 297 | 298 | if __name__ == "__main__": 299 | ClouderaManagerInventory() 300 | -------------------------------------------------------------------------------- /group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ansible_become: true 4 | tmp_dir: /tmp 5 | 6 | ## ------------------------------------------------------------------------------------------------------------ 7 | ## Security options 8 | ## ------------------------------------------------------------------------------------------------------------ 9 | 10 | krb5_realm: MIT.EXAMPLE.COM 11 | # 'Active Directory', 'MIT KDC', or 'none' to disable security 12 | krb5_kdc_type: MIT KDC 13 | krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" 14 | krb5_kdc_admin_passwd: changeme 15 | krb5_kdc_master_passwd: changeme 16 | 17 | # krb5_realm: AD.EXAMPLE.COM 18 | # krb5_kdc_type: Active Directory 19 | # krb5_kdc_host: my.ad.example.com 20 | # krb5_kdc_admin_user: "admin@{{ krb5_realm }}" 21 | # krb5_kdc_admin_passwd: changeme 22 | # krb5_ad_suffix: "OU=Clusters,DC=ad,DC=example,DC=com" 23 | # krb5_ad_delete_on_regenerate: true 24 | 25 | # krb5_cm_managed_krb5_conf: false 26 | # krb5_enc_types: "aes256-cts aes128-cts rc4-hmac" 27 | 28 | ## ------------------------------------------------------------------------------------------------------------ 29 | ## SSSD installation options 30 | ## ------------------------------------------------------------------------------------------------------------ 31 | 32 | ad_domain: "{{ krb5_realm.lower() }}" 33 | computer_ou: OU=computer_hosts,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com 34 | ldap_group_search_base: OU=groups,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com 35 | ldap_user_search_base: DC=ad,DC=sec,DC=example,DC=com?subtree?(memberOf=CN=hadoop_users,OU=groups,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com) 36 | override_gid: 999999 37 | ad_site: Default-First-Site-Name 38 | 39 | ## ------------------------------------------------------------------------------------------------------------ 40 | ## Cluster software installation options 41 | ## ------------------------------------------------------------------------------------------------------------ 42 | 43 | # Version of CDH to install 44 | cluster_version_cdh: 6.3.2 45 | #cluster_version_cdh: 7.x 46 | 47 | # Version of Cloudera Manager to install 48 | cluster_version_cm: 6.3.1 49 | #cluster_version_cm: "{{ cluster_version_cdh }}" 50 | #cluster_version_cm: 7.x.0 51 | 52 | # Version of CDS Powered by Apache Spark (note: not installed if CDH6/7 is also selected) 53 | cluster_version_cds: 2.4.0.cloudera2 54 | 55 | # Helper variables for major and minor versions 56 | cluster_version_cdh_major: "{{ cluster_version_cdh.split('.')[0] }}" 57 | cluster_version_cdh_minor: "{{ cluster_version_cdh.split('.')[1] }}" 58 | cluster_version_cm_major: "{{ cluster_version_cm.split('.')[0] }}" 59 | cluster_version_cm_minor: "{{ cluster_version_cm.split('.')[1] }}" 60 | 61 | cloudera_archive_protocol: https:// 62 | cloudera_archive: archive.cloudera.com 63 | cloudera_archive_authn: "" 64 | 65 | configs_by_version: 66 | "5": 67 | scm_repo_url: "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm5/redhat/{{ ansible_distribution_major_version }}/x86_64/cm/{{ cluster_version_cm }}/" 68 | scm_repo_gpgkey: "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm5/redhat/{{ ansible_distribution_major_version }}/x86_64/cm/RPM-GPG-KEY-cloudera" 69 | scm_parcel_repositories: 70 | - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cdh5/parcels/{{ cluster_version_cdh }}/" 71 | - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/spark2/parcels/{{ cluster_version_cds }}/" 72 | scm_csds: 73 | - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/spark2/csd/SPARK2_ON_YARN-{{cluster_version_cds}}.jar" 74 | scm_prepare_database_script_path: "/usr/share/cmf/schema/scm_prepare_database.sh" 75 | "6": 76 | scm_repo_url: "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm6/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum" 77 | scm_repo_gpgkey: "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm6/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum/RPM-GPG-KEY-cloudera" 78 | scm_parcel_repositories: 79 | - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cdh6/{{ cluster_version_cdh }}/parcels" 80 | scm_prepare_database_script_path: "/opt/cloudera/cm/schema/scm_prepare_database.sh" 81 | "7": 82 | scm_repo_url: "{{ cloudera_archive_protocol }}{{ cloudera_archive_authn }}@{{ cloudera_archive }}/p/cm7/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum" 83 | scm_repo_gpgkey: "{{ cloudera_archive_protocol }}{{ cloudera_archive_authn }}@{{ cloudera_archive }}/p/cm7/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum/RPM-GPG-KEY-cloudera" 84 | scm_parcel_repositories: 85 | - "{{ cloudera_archive_protocol }}{{ cloudera_archive_authn }}@{{ cloudera_archive }}/p/cdh7/{{ cluster_version_cdh }}/parcels" 86 | scm_prepare_database_script_path: "/opt/cloudera/cm/schema/scm_prepare_database.sh" 87 | 88 | scm_default_user: admin 89 | scm_default_pass: admin 90 | scm_port: 7180 91 | scm_license_file: /path/to/cloudera_license.txt 92 | scm_parcel_repositories: "{{ configs_by_version[cluster_version_cdh_major].scm_parcel_repositories }}" 93 | scm_prepare_database_script_path: "{{ configs_by_version[cluster_version_cm_major].scm_prepare_database_script_path }}" 94 | scm_repo_url: "{{ configs_by_version[cluster_version_cm_major].scm_repo_url }}" 95 | scm_repo_gpgkey: "{{ configs_by_version[cluster_version_cm_major].scm_repo_gpgkey }}" 96 | scm_csds: "{{ configs_by_version[cluster_version_cm_major].scm_csds | default([]) }}" 97 | 98 | ## ------------------------------------------------------------------------------------------------------------ 99 | ## Java installation options 100 | ## ------------------------------------------------------------------------------------------------------------ 101 | 102 | java_installation_strategy: package # can be set to 'none', 'package' or 'rpm' 103 | 104 | java_package: java-1.8.0-openjdk-devel 105 | java_rpm_location: /tmp/jdk-8u181-linux-x64.rpm 106 | java_rpm_remote_src: no 107 | java_jce_location: /tmp/jce_policy-8.zip 108 | java_jce_remote_src: no 109 | -------------------------------------------------------------------------------- /group_vars/cdh_servers.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | db_hostname: "{{ hostvars[groups['db_server'][0]]['inventory_hostname'] }}" 4 | scm_hostname: "{{ hostvars[groups['scm_server'][0]]['inventory_hostname'] }}" 5 | 6 | cluster_display_name: cluster_1 7 | 8 | cdh_services: 9 | 10 | - type: hbase 11 | 12 | - type: hdfs 13 | dfs_data_dir_list: /dfs/dn 14 | fs_checkpoint_dir_list: /dfs/snn 15 | dfs_name_dir_list: /dfs/nn 16 | dfs_journalnode_edits_dir: /dfs/jn 17 | 18 | - type: hive 19 | 20 | - type: hue 21 | 22 | - type: impala 23 | scratch_dirs: /tmp/impala 24 | 25 | - type: oozie 26 | 27 | - type: sentry 28 | 29 | - type: spark 30 | 31 | - type: yarn 32 | yarn_nodemanager_local_dirs: /tmp/nm 33 | yarn_nodemanager_log_dirs: /var/log/nm 34 | 35 | - type: zookeeper 36 | -------------------------------------------------------------------------------- /group_vars/db_server.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | mysql_datadir: /var/lib/mysql 4 | mysql_socket: /var/lib/mysql/mysql.sock 5 | mysql_port: 3306 6 | mysql_log_bin: /var/lib/mysql/mysql_binary_log 7 | mysql_log: /var/log/mariadb/mariadb.log 8 | mysql_pid_dir: /var/run/mariadb 9 | mysql_pid_file: "{{ mysql_pid_dir }}/mariadb.pid" 10 | mysql_innodb_log_buffer_size: 64M 11 | mysql_innodb_buffer_pool_size: 4G 12 | mysql_root_password: changeme 13 | 14 | # Supported database types: mysql,postgresql,... 15 | database_type: mysql 16 | 17 | databases: 18 | scm: 19 | name: 'scm' 20 | user: 'scm' 21 | pass: 'scm_password' 22 | type: '{{ database_type }}' 23 | amon: 24 | name: 'amon' 25 | user: 'amon' 26 | pass: 'amon_password' 27 | type: '{{ database_type }}' 28 | rman: 29 | name: 'rman' 30 | user: 'rman' 31 | pass: 'rman_password' 32 | type: '{{ database_type }}' 33 | nav: 34 | name: 'nav' 35 | user: 'nav' 36 | pass: 'nav_password' 37 | type: '{{ database_type }}' 38 | navms: 39 | name: 'navms' 40 | user: 'navms' 41 | pass: 'navms_password' 42 | type: '{{ database_type }}' 43 | metastore: 44 | name: 'metastore' 45 | user: 'hive' 46 | pass: 'hive_password' 47 | type: '{{ database_type }}' 48 | sentry: 49 | name: 'sentry' 50 | user: 'sentry' 51 | pass: 'sentry_password' 52 | type: '{{ database_type }}' 53 | hue: 54 | name: 'hue' 55 | user: 'hue' 56 | pass: 'hue_password' 57 | type: '{{ database_type }}' 58 | oozie: 59 | name: 'oozie' 60 | user: 'oozie' 61 | pass: 'oozie_password' 62 | type: '{{ database_type }}' 63 | das: 64 | name: 'das' 65 | user: 'das' 66 | pass: 'das_password' 67 | type: '{{ database_type }}' 68 | ranger: 69 | name: 'ranger' 70 | user: 'ranger' 71 | pass: 'ranger_password' 72 | type: '{{ database_type }}' 73 | -------------------------------------------------------------------------------- /hosts: -------------------------------------------------------------------------------- 1 | [scm_server] 2 | 3 | 4 | [db_server] 5 | 6 | 7 | [krb5_server] 8 | 9 | 10 | [utility_servers:children] 11 | scm_server 12 | db_server 13 | krb5_server 14 | 15 | [gateway_servers] 16 | host_template=HostTemplate-Gateway 17 | 18 | [edge_servers] 19 | host_template=HostTemplate-Edge role_ref_names=HDFS-HTTPFS-1 20 | 21 | [master_servers] 22 | host_template=HostTemplate-Master1 23 | host_template=HostTemplate-Master2 24 | host_template=HostTemplate-Master3 25 | 26 | [worker_servers] 27 | 28 | 29 | 30 | 31 | [worker_servers:vars] 32 | host_template=HostTemplate-Workers 33 | 34 | [cdh_servers:children] 35 | utility_servers 36 | gateway_servers 37 | master_servers 38 | worker_servers 39 | 40 | [all:vars] 41 | ansible_user=ec2-user 42 | -------------------------------------------------------------------------------- /icla/Cloudera_CCLA_25APR2018.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cloudera-playbook/82698556b7a1a3985c6673254e05d1e51a3db694/icla/Cloudera_CCLA_25APR2018.pdf -------------------------------------------------------------------------------- /icla/Cloudera_ICLA_25APR2018.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cloudera-playbook/82698556b7a1a3985c6673254e05d1e51a3db694/icla/Cloudera_ICLA_25APR2018.pdf -------------------------------------------------------------------------------- /images/figure_1_ansible_inventory-v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/cloudera-playbook/82698556b7a1a3985c6673254e05d1e51a3db694/images/figure_1_ansible_inventory-v2.png -------------------------------------------------------------------------------- /realm_join.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | vars_prompt: 4 | - name: "bind_user" 5 | prompt: "bind user" 6 | private: no 7 | 8 | - name: "bind_password" 9 | prompt: "bind password" 10 | private: yes 11 | roles: 12 | - realm/join 13 | -------------------------------------------------------------------------------- /realm_leave.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | roles: 4 | - realm/leave 5 | -------------------------------------------------------------------------------- /roles/cdh/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - include_vars: ../../../group_vars/cdh_servers.yml 4 | - include_vars: ../../../group_vars/db_server.yml 5 | 6 | # Check whether cluster already exists 7 | # https://cloudera.github.io/cm_api/apidocs/v13/path__clusters.html 8 | - name: Check whether cluster exists 9 | uri: 10 | url: "{{ cm_api_url }}/clusters/{{ cluster_display_name }}" 11 | method: GET 12 | status_code: 200,404 13 | force_basic_auth: yes 14 | user: "{{ scm_default_user }}" 15 | password: "{{ scm_default_pass }}" 16 | return_content: yes 17 | register: clusters_resp 18 | 19 | - set_fact: 20 | cluster_exists: "{{ 'True' if clusters_resp.status == 200 else 'False' }}" 21 | 22 | - debug: 23 | msg: "Cluster '{{ cluster_display_name }}' exists - {{ cluster_exists }}" 24 | verbosity: 1 25 | 26 | - name: Discover product versions from parcel manifests 27 | uri: 28 | url: "{{ item }}/manifest.json" 29 | status_code: 200 30 | body_format: json 31 | return_content: yes 32 | register: manifests 33 | with_items: 34 | - "{{ scm_parcel_repositories }}" 35 | 36 | - set_fact: 37 | scm_products: "{{ manifests.results | map(attribute='json') | list | json_query('[*].parcels[0].parcelName') | map('regex_replace', '-[a-z0-9]+.parcel$','') | list }}" 38 | 39 | # https://www.cloudera.com/documentation/enterprise/latest/topics/install_cluster_template.html 40 | - set_fact: 41 | cluster_template_json: "{{ lookup('template', 'base.j2', convert_data=False) }}" 42 | when: not cluster_exists 43 | 44 | - debug: 45 | var: cluster_template_json 46 | verbosity: 1 47 | 48 | # https://cloudera.github.io/cm_api/apidocs/v13/path__cm_importClusterTemplate.html 49 | - name: Import cluster template 50 | uri: 51 | url: "{{ cm_api_url }}/cm/importClusterTemplate?addRepositories=true" 52 | method: POST 53 | body_format: json 54 | body: "{{ cluster_template_json|to_json }}" 55 | status_code: 200 56 | force_basic_auth: yes 57 | user: "{{ scm_default_user }}" 58 | password: "{{ scm_default_pass }}" 59 | return_content: yes 60 | register: template_resp 61 | when: not cluster_exists 62 | 63 | - debug: 64 | var: template_resp 65 | verbosity: 1 66 | when: not cluster_exists 67 | 68 | - set_fact: 69 | command_id: "{{ template_resp.json.id }}" 70 | when: not cluster_exists 71 | 72 | - debug: 73 | msg: "Login to Cloudera Manager to monitor import progress - http://{{ hostvars[scm_hostname]['inventory_hostname'] }}:{{ scm_port }}/cmf/command/{{ command_id }}/details" 74 | verbosity: 1 75 | when: not cluster_exists 76 | 77 | - name: Wait for import cluster template command to complete 78 | uri: 79 | url: "{{ cm_api_url }}/commands/{{ command_id }}" 80 | body_format: json 81 | force_basic_auth: yes 82 | user: "{{ scm_default_user }}" 83 | password: "{{ scm_default_pass }}" 84 | return_content: yes 85 | register: result 86 | until: not result.json.active 87 | retries: 10 88 | delay: 60 89 | when: not cluster_exists 90 | failed_when: "'Failed to import cluster template' in result.content" 91 | 92 | - debug: 93 | var: result 94 | when: not cluster_exists 95 | 96 | - name: Restart Cloudera Management Service 97 | uri: 98 | url: "{{ cm_api_url }}/cm/service/commands/restart" 99 | method: POST 100 | status_code: 200 101 | force_basic_auth: yes 102 | user: "{{ scm_default_user }}" 103 | password: "{{ scm_default_pass }}" 104 | return_content: yes 105 | register: start_resp 106 | failed_when: "'startTime' not in start_resp.content" 107 | when: not cluster_exists 108 | -------------------------------------------------------------------------------- /roles/cdh/templates/base.j2: -------------------------------------------------------------------------------- 1 | { 2 | "cdhVersion" : "{{ cluster_version_cdh }}", 3 | "displayName" : "{{ cluster_display_name }}", 4 | "cmVersion" : "{{ cluster_version_cm }}", 5 | "products" : [ 6 | {% set prod_j = joiner(",") %} 7 | {% for product in scm_products %} 8 | {{ prod_j() }} 9 | { 10 | "product" : "{{ product.split('-')[0] }}", 11 | "version" : "{{ '-'.join(product.split('-')[1:]) }}" 12 | } 13 | {% endfor %} 14 | ], 15 | "services" : [ 16 | {% set serv_j = joiner(",") %} 17 | {% for service in cdh_services %} 18 | {{ serv_j() }} 19 | {% set t = service['type'] + '.j2' %}{% include t %} 20 | {% endfor %} 21 | ], 22 | "hostTemplates" : {% include 'host.j2' %}, 23 | "instantiator" : {% include 'instantiator.j2' %} 24 | } 25 | -------------------------------------------------------------------------------- /roles/cdh/templates/hbase.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "HBASE-1", 3 | "serviceType": "HBASE", 4 | "serviceConfigs": [ 5 | { 6 | "name": "hdfs_service", 7 | "ref": "HDFS-1" 8 | }, { 9 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 10 | "name" : "hbase_restserver_security_authentication", 11 | "value" : "kerberos", 12 | }, { 13 | "name" : "hbase_security_authentication", 14 | "value" : "kerberos", 15 | }, { 16 | {% endif %} 17 | "name": "zookeeper_service", 18 | "ref": "ZOOKEEPER-1" 19 | } 20 | ], 21 | "roleConfigGroups": [ 22 | { 23 | "refName": "HBASE-1-MASTER-BASE", 24 | "roleType": "MASTER", 25 | "configs": [], 26 | "base": true 27 | }, { 28 | "refName": "HBASE-1-REGIONSERVER-BASE", 29 | "roleType": "REGIONSERVER", 30 | "configs": [], 31 | "base": true 32 | }, { 33 | "refName": "HBASE-1-HBASETHRIFTSERVER-BASE", 34 | "roleType": "HBASETHRIFTSERVER", 35 | "configs": [], 36 | "base": true 37 | }, { 38 | "refName": "HBASE-1-HBASERESTSERVER-BASE", 39 | "roleType": "HBASERESTSERVER", 40 | "configs": [], 41 | "base": true 42 | } 43 | ] 44 | } -------------------------------------------------------------------------------- /roles/cdh/templates/hdfs.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "HDFS-1", 3 | "serviceType": "HDFS", 4 | "serviceConfigs": [ 5 | { 6 | "name": "dfs_namenode_acls_enabled", 7 | "value": "true" 8 | }, { 9 | "name": "hadoop_security_authorization", 10 | "value": "true" 11 | }, { 12 | "name": "hadoop_security_authentication", 13 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 14 | "value": "kerberos" 15 | {% else %} 16 | "value": "simple" 17 | {% endif %} 18 | }, { 19 | "name": "dfs_ha_fencing_methods", 20 | "value": "shell(true)" 21 | }, { 22 | "name": "hadoop_secure_web_ui", 23 | "value": "true" 24 | }, { 25 | "name": "zookeeper_service", 26 | "ref": "ZOOKEEPER-1" 27 | } 28 | ], 29 | "roleConfigGroups": [ 30 | { 31 | "refName": "HDFS-1-FAILOVERCONTROLLER-BASE", 32 | "roleType": "FAILOVERCONTROLLER", 33 | "configs": [], 34 | "base": true 35 | }, { 36 | "refName": "HDFS-1-DATANODE-BASE", 37 | "roleType": "DATANODE", 38 | "configs": [ 39 | { 40 | "name": "dfs_data_dir_list", 41 | "variable": "DFS_DATA_DIR_LIST" 42 | } 43 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 44 | , { 45 | "name": "dfs_datanode_port", 46 | "value": "1004" 47 | }, { 48 | "name": "dfs_datanode_http_port", 49 | "value": "1006" 50 | } 51 | {% endif %} 52 | ], 53 | "base": true 54 | }, { 55 | "refName": "HDFS-1-SECONDARYNAMENODE-BASE", 56 | "roleType": "SECONDARYNAMENODE", 57 | "configs": [ 58 | { 59 | "name": "fs_checkpoint_dir_list", 60 | "variable": "FS_CHECKPOINT_DIR_LIST" 61 | } 62 | ], 63 | "base": true 64 | }, { 65 | "refName": "HDFS-1-NAMENODE-BASE", 66 | "roleType": "NAMENODE", 67 | "configs": [ 68 | { 69 | "name": "dfs_federation_namenode_nameservice", 70 | "value": "nameservice1" 71 | }, { 72 | "name": "dfs_namenode_quorum_journal_name", 73 | "value": "nameservice1" 74 | }, { 75 | "name": "autofailover_enabled", 76 | "value": "true" 77 | }, { 78 | "name": "dfs_name_dir_list", 79 | "variable": "DFS_NAME_DIR_LIST" 80 | } 81 | ], 82 | "base": true 83 | }, { 84 | "refName": "HDFS-1-JOURNALNODE-BASE", 85 | "roleType": "JOURNALNODE", 86 | "configs": [ 87 | { 88 | "name": "dfs_journalnode_edits_dir", 89 | "variable": "DFS_JOURNALNODE_EDITS_DIR" 90 | } 91 | ], 92 | "base": true 93 | }, { 94 | "refName": "HDFS-1-HTTPFS-BASE", 95 | "roleType": "HTTPFS", 96 | "configs": [], 97 | "base": true 98 | }, { 99 | "refName": "HDFS-1-NFSGATEWAY-BASE", 100 | "roleType": "NFSGATEWAY", 101 | "configs": [], 102 | "base": true 103 | }, { 104 | "refName": "HDFS-1-BALANCER-BASE", 105 | "roleType": "BALANCER", 106 | "configs": [], 107 | "base": true 108 | }, { 109 | "refName": "HDFS-1-GATEWAY-BASE", 110 | "roleType": "GATEWAY", 111 | "configs": [ 112 | { 113 | "name": "dfs_client_use_trash", 114 | "value": "true" 115 | } 116 | ], 117 | "base": true 118 | } 119 | ], 120 | "roles": [ 121 | { 122 | "refName": "HDFS-HTTPFS-1", 123 | "roleType": "HTTPFS" 124 | } 125 | ] 126 | } 127 | -------------------------------------------------------------------------------- /roles/cdh/templates/hive.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "HIVE-1", 3 | "serviceType": "HIVE", 4 | "serviceConfigs": [ 5 | { 6 | "name": "hive_metastore_database_host", 7 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 8 | }, { 9 | "name": "hive_metastore_database_type", 10 | "value": "{{ databases.metastore.type }}" 11 | }, { 12 | "name": "hive_metastore_database_name", 13 | "value": "{{ databases.metastore.name }}" 14 | }, { 15 | "name": "hive_metastore_database_user", 16 | "value": "{{ databases.metastore.user }}" 17 | }, { 18 | "name": "hive_metastore_database_password", 19 | "value": "{{ databases.metastore.pass }}" 20 | }, { 21 | "name": "spark_on_yarn_service", 22 | "ref": "SPARK_ON_YARN-1" 23 | }, { 24 | "name": "mapreduce_yarn_service", 25 | "ref": "YARN-1" 26 | }, { 27 | "name": "zookeeper_service", 28 | "ref": "ZOOKEEPER-1" 29 | }, { 30 | "name": "sentry_service", 31 | "ref": "SENTRY-1" 32 | } 33 | ], 34 | "roleConfigGroups": [ 35 | { 36 | "refName": "HIVE-1-GATEWAY-BASE", 37 | "roleType": "GATEWAY", 38 | "configs": [], 39 | "base": true 40 | }, { 41 | "refName": "HIVE-1-WEBHCAT-BASE", 42 | "roleType": "WEBHCAT", 43 | "configs": [ 44 | { 45 | "name": "hive_webhcat_address_port", 46 | "value": "7272" 47 | } 48 | ], 49 | "base": true 50 | }, { 51 | "refName": "HIVE-1-HIVEMETASTORE-BASE", 52 | "roleType": "HIVEMETASTORE", 53 | "configs": [ 54 | { 55 | "name": "hive_enable_db_notification", 56 | "value": "true" 57 | }, { 58 | "name": "hive_metastore_delegation_token_store", 59 | "value": "org.apache.hadoop.hive.thrift.DBTokenStore" 60 | } 61 | ], 62 | "base": true 63 | }, { 64 | "refName": "HIVE-1-HIVESERVER2-BASE", 65 | "roleType": "HIVESERVER2", 66 | "configs": [ 67 | { 68 | "name": "hiveserver2_enable_impersonation", 69 | "value": "false" 70 | } 71 | ], 72 | "base": true 73 | } 74 | ] 75 | } -------------------------------------------------------------------------------- /roles/cdh/templates/host.j2: -------------------------------------------------------------------------------- 1 | [ { 2 | "refName": "HostTemplate-Gateway", 3 | "cardinality": 1, 4 | "roleConfigGroupsRefNames": [ 5 | "HDFS-1-GATEWAY-BASE", 6 | "HIVE-1-GATEWAY-BASE", 7 | "SPARK_ON_YARN-1-GATEWAY-BASE", 8 | "YARN-1-GATEWAY-BASE" ] 9 | }, { 10 | "refName": "HostTemplate-Edge", 11 | "cardinality": 1, 12 | "roleConfigGroupsRefNames": [ 13 | "HDFS-1-HTTPFS-BASE", 14 | "HBASE-1-HBASERESTSERVER-BASE", 15 | "HBASE-1-HBASETHRIFTSERVER-BASE", 16 | "HIVE-1-HIVESERVER2-BASE", 17 | "HUE-1-HUE_SERVER-BASE", 18 | "HUE-1-HUE_LOAD_BALANCER-BASE", 19 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 20 | "HUE-1-KT_RENEWER-BASE", 21 | {% endif %} 22 | "OOZIE-1-OOZIE_SERVER-BASE", 23 | "SPARK_ON_YARN-1-GATEWAY-BASE", 24 | "YARN-1-GATEWAY-BASE" ] 25 | }, { 26 | "refName": "HostTemplate-Master1", 27 | "cardinality": 1, 28 | "roleConfigGroupsRefNames": [ 29 | "HBASE-1-MASTER-BASE", 30 | "HDFS-1-NAMENODE-BASE", 31 | "HDFS-1-FAILOVERCONTROLLER-BASE", 32 | "HDFS-1-JOURNALNODE-BASE", 33 | "HIVE-1-HIVEMETASTORE-BASE", 34 | "SENTRY-1-SENTRY_SERVER-BASE", 35 | "YARN-1-RESOURCEMANAGER-BASE", 36 | "ZOOKEEPER-1-SERVER-BASE" ] 37 | }, { 38 | "refName": "HostTemplate-Master2", 39 | "cardinality": 1, 40 | "roleConfigGroupsRefNames" : [ 41 | "HBASE-1-MASTER-BASE", 42 | "HDFS-1-NAMENODE-BASE", 43 | "HDFS-1-FAILOVERCONTROLLER-BASE", 44 | "HDFS-1-JOURNALNODE-BASE", 45 | "HIVE-1-HIVEMETASTORE-BASE", 46 | "SENTRY-1-SENTRY_SERVER-BASE", 47 | "YARN-1-RESOURCEMANAGER-BASE", 48 | "ZOOKEEPER-1-SERVER-BASE" ] 49 | }, { 50 | "refName": "HostTemplate-Master3", 51 | "cardinality": 1, 52 | "roleConfigGroupsRefNames": [ 53 | "HBASE-1-MASTER-BASE", 54 | "HDFS-1-BALANCER-BASE", 55 | "HDFS-1-JOURNALNODE-BASE", 56 | "IMPALA-1-CATALOGSERVER-BASE", 57 | "IMPALA-1-STATESTORE-BASE", 58 | "SPARK_ON_YARN-1-SPARK_YARN_HISTORY_SERVER-BASE", 59 | {% if cluster_version_cdh_major < '6' %} 60 | "SPARK2_ON_YARN-1-SPARK2_YARN_HISTORY_SERVER-BASE", 61 | {% endif %} 62 | "YARN-1-JOBHISTORY-BASE", 63 | "ZOOKEEPER-1-SERVER-BASE" ] 64 | }, { 65 | "refName": "HostTemplate-Workers", 66 | "cardinality": 3, 67 | "roleConfigGroupsRefNames": [ 68 | "HBASE-1-REGIONSERVER-BASE", 69 | "HDFS-1-DATANODE-BASE", 70 | "HIVE-1-GATEWAY-BASE", 71 | "IMPALA-1-IMPALAD-BASE", 72 | "SPARK_ON_YARN-1-GATEWAY-BASE", 73 | {% if cluster_version_cdh_major < '6' %} 74 | "SPARK2_ON_YARN-1-GATEWAY-BASE", 75 | {% endif %} 76 | "YARN-1-NODEMANAGER-BASE" ] 77 | } ] -------------------------------------------------------------------------------- /roles/cdh/templates/hue.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "HUE-1", 3 | "serviceType": "HUE", 4 | "serviceConfigs": [ 5 | { 6 | "name": "database_host", 7 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 8 | }, { 9 | "name": "database_type", 10 | "value": "{{ databases.hue.type }}" 11 | }, { 12 | "name": "database_name", 13 | "value": "{{ databases.hue.name }}" 14 | }, { 15 | "name": "database_user", 16 | "value": "{{ databases.hue.user }}" 17 | }, { 18 | "name": "database_password", 19 | "value": "{{ databases.hue.pass }}" 20 | }, { 21 | "name": "oozie_service", 22 | "ref": "OOZIE-1" 23 | }, { 24 | "name": "impala_service", 25 | "ref": "IMPALA-1" 26 | }, { 27 | "name": "hive_service", 28 | "ref": "HIVE-1" 29 | }, { 30 | "name": "sentry_service", 31 | "ref": "SENTRY-1" 32 | }, { 33 | "name": "hue_service_safety_valve", 34 | "value": "[impala]\nserver_port=21051\n\n[beeswax]\ndownload_cell_limit=10" 35 | }, { 36 | "name": "zookeeper_service", 37 | "ref": "ZOOKEEPER-1" 38 | }, { 39 | "name": "hue_webhdfs", 40 | "ref": "HDFS-HTTPFS-1" 41 | } 42 | ], 43 | "roleConfigGroups": [ 44 | { 45 | "refName": "HUE-1-HUE_SERVER-BASE", 46 | "roleType": "HUE_SERVER", 47 | "configs": [], 48 | "base": true 49 | }, { 50 | "refName": "HUE-1-HUE_LOAD_BALANCER-BASE", 51 | "roleType": "HUE_LOAD_BALANCER", 52 | "configs": [], 53 | "base": true 54 | } 55 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 56 | , { 57 | "refName": "HUE-1-KT_RENEWER-BASE", 58 | "roleType": "KT_RENEWER", 59 | "configs": [], 60 | "base": true 61 | } 62 | {% endif %} 63 | ] 64 | } -------------------------------------------------------------------------------- /roles/cdh/templates/impala.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "IMPALA-1", 3 | "serviceType": "IMPALA", 4 | "serviceConfigs": [ 5 | { 6 | "name": "impala_cmd_args_safety_valve", 7 | "value": "" 8 | }, { 9 | "name": "hdfs_service", 10 | "ref": "HDFS-1" 11 | }, { 12 | "name": "admission_control_enabled", 13 | "value": "true" 14 | }, { 15 | "name": "sentry_service", 16 | "ref": "SENTRY-1" 17 | }, { 18 | "name": "kerberos_reinit_interval", 19 | "value": "10" 20 | }, { 21 | "name": "enable_core_dump", 22 | "value": "true" 23 | }, { 24 | "name": "hive_service", 25 | "ref": "HIVE-1" 26 | }, { 27 | "name": "all_admission_control_enabled", 28 | "value": "true" 29 | } 30 | ], 31 | "roleConfigGroups": [ 32 | { 33 | "refName": "IMPALA-1-IMPALAD-BASE", 34 | "roleType": "IMPALAD", 35 | "configs": [ 36 | { 37 | "name": "enable_audit_event_log", 38 | "value": "true" 39 | }, { 40 | "name": "scratch_dirs", 41 | "variable": "SCRATCH_DIRS" 42 | }, { 43 | "name": "logbuflevel", 44 | "value": "-1" 45 | } 46 | ], 47 | "base": true 48 | }, { 49 | "refName": "IMPALA-1-CATALOGSERVER-BASE", 50 | "roleType": "CATALOGSERVER", 51 | "configs": [ 52 | { 53 | "name": "logbuflevel", 54 | "value": "-1" 55 | }, { 56 | "name": "catalogd_embedded_jvm_heapsize", 57 | "value": "603979776" 58 | } 59 | ], 60 | "base": true 61 | }, { 62 | "refName": "IMPALA-1-STATESTORE-BASE", 63 | "roleType": "STATESTORE", 64 | "configs": [ 65 | { 66 | "name": "logbuflevel", 67 | "value": "-1" 68 | } 69 | ], 70 | "base": true 71 | } 72 | ] 73 | } -------------------------------------------------------------------------------- /roles/cdh/templates/instantiator.j2: -------------------------------------------------------------------------------- 1 | { 2 | "clusterName" : "{{ cluster_display_name }}", 3 | "hosts" : [ 4 | {% set host_joiner = joiner(",") %} 5 | {% for host in groups['cdh_servers'] %} 6 | {% if 'host_template' in hostvars[host] %} 7 | {{ host_joiner() }} 8 | { 9 | "hostName" : "{{ scm_hosts['names'][host] }}", 10 | "hostTemplateRefName" : "{{ hostvars[host]['host_template'] }}" 11 | {% if 'role_ref_names' in hostvars[host] %} 12 | ,"roleRefNames" : [ "{{ hostvars[host]['role_ref_names'] }}" ] 13 | {% endif %} 14 | } 15 | {% endif %} 16 | {% endfor %} 17 | ], 18 | "variables" : [ 19 | {% set var_joiner = joiner(",") %} 20 | {% for item in cdh_services %} 21 | {% for (k,v) in item.items() %} 22 | {% if not k|lower == 'type' %} 23 | {{ var_joiner() }} 24 | { 25 | "name": "{{ k|upper }}", 26 | "value": "{{ v }}" 27 | } 28 | {% endif %} 29 | {% endfor %} 30 | {% endfor %} 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /roles/cdh/templates/oozie.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "OOZIE-1", 3 | "serviceType": "OOZIE", 4 | "serviceConfigs": [ 5 | { 6 | "name": "oozie_upload_sharelib_cmd_timeout", 7 | "value": "540" 8 | }, { 9 | "name": "hive_service", 10 | "ref": "HIVE-1" 11 | }, { 12 | "name": "mapreduce_yarn_service", 13 | "ref": "YARN-1" 14 | }, { 15 | "name": "spark_on_yarn_service", 16 | "ref": "SPARK_ON_YARN-1" 17 | }, { 18 | "name": "zookeeper_service", 19 | "ref": "ZOOKEEPER-1" 20 | } 21 | ], 22 | "roleConfigGroups": [ 23 | { 24 | "refName": "OOZIE-1-OOZIE_SERVER-BASE", 25 | "roleType": "OOZIE_SERVER", 26 | "configs": [ 27 | { 28 | "name": "oozie_database_host", 29 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 30 | }, { 31 | "name": "oozie_database_type", 32 | "value": "{{ databases.oozie.type }}" 33 | }, { 34 | "name": "oozie_database_name", 35 | "value": "{{ databases.oozie.name }}" 36 | }, { 37 | "name": "oozie_database_user", 38 | "value": "{{ databases.oozie.user }}" 39 | }, { 40 | "name": "oozie_database_password", 41 | "value": "{{ databases.oozie.pass }}" 42 | }, { 43 | "name": "oozie_workflow_extension_schemas", 44 | "value": ",ssh-action-0.1.xsd,hive-action-0.3.xsd,sqoop-action-0.3.xsd,shell-action-0.2.xsd,shell-action-0.1.xsd" 45 | }, { 46 | "name": "oozie_config_safety_valve", 47 | "value": "\n\noozie.action.launcher.mapreduce.job.ubertask.enable\nfalse\n" 48 | } 49 | ], 50 | "base": true 51 | } 52 | ] 53 | } -------------------------------------------------------------------------------- /roles/cdh/templates/sentry.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "SENTRY-1", 3 | "serviceType": "SENTRY", 4 | "serviceConfigs": [ 5 | { 6 | "name": "sentry_server_database_host", 7 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 8 | }, { 9 | "name": "sentry_server_database_type", 10 | "value": "{{ databases.sentry.type }}" 11 | }, { 12 | "name": "sentry_server_database_name", 13 | "value": "{{ databases.sentry.name }}" 14 | }, { 15 | "name": "sentry_server_database_user", 16 | "value": "{{ databases.sentry.user }}" 17 | }, { 18 | "name": "sentry_server_database_password", 19 | "value": "{{ databases.sentry.pass }}" 20 | }, { 21 | "name": "zookeeper_service", 22 | "ref": "ZOOKEEPER-1" 23 | }, { 24 | "name": "hdfs_service", 25 | "ref": "HDFS-1" 26 | } 27 | ], 28 | "roleConfigGroups": [ 29 | { 30 | "refName": "SENTRY-1-GATEWAY-BASE", 31 | "roleType": "GATEWAY", 32 | "base": true 33 | }, { 34 | "refName": "SENTRY-1-SENTRY_SERVER-BASE", 35 | "roleType": "SENTRY_SERVER", 36 | "configs": [], 37 | "base": true 38 | } 39 | ] 40 | } -------------------------------------------------------------------------------- /roles/cdh/templates/spark.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "SPARK_ON_YARN-1", 3 | "serviceType": "SPARK_ON_YARN", 4 | "serviceConfigs": [ 5 | { 6 | "name": "yarn_service", 7 | "ref": "YARN-1" 8 | } 9 | ], 10 | "roleConfigGroups": [ 11 | { 12 | "refName": "SPARK_ON_YARN-1-GATEWAY-BASE", 13 | "roleType": "GATEWAY", 14 | "base": true 15 | }, { 16 | "refName": "SPARK_ON_YARN-1-SPARK_YARN_HISTORY_SERVER-BASE", 17 | "roleType": "SPARK_YARN_HISTORY_SERVER", 18 | "configs": [], 19 | "base": true 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /roles/cdh/templates/spark2.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "SPARK2_ON_YARN-1", 3 | "serviceType": "SPARK2_ON_YARN", 4 | "serviceConfigs": [ 5 | { 6 | "name": "yarn_service", 7 | "ref": "YARN-1" 8 | } 9 | ], 10 | "roleConfigGroups": [ 11 | { 12 | "refName": "SPARK2_ON_YARN-1-GATEWAY-BASE", 13 | "roleType": "GATEWAY", 14 | "base": true 15 | }, { 16 | "refName": "SPARK2_ON_YARN-1-SPARK2_YARN_HISTORY_SERVER-BASE", 17 | "roleType": "SPARK2_YARN_HISTORY_SERVER", 18 | "configs": [], 19 | "base": true 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /roles/cdh/templates/yarn.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "YARN-1", 3 | "serviceType": "YARN", 4 | "serviceConfigs": [ 5 | { 6 | "name": "cm_yarn_container_usage_job_user", 7 | "value": "cmjobuser" 8 | }, { 9 | "name": "zookeeper_service", 10 | "ref": "ZOOKEEPER-1" 11 | }, { 12 | "name": "hdfs_service", 13 | "ref": "HDFS-1" 14 | }, { 15 | "name": "hadoop_secure_web_ui", 16 | "value": "true" 17 | }, { 18 | "name": "cm_yarn_enable_container_usage_aggregation", 19 | "value": "true" 20 | } 21 | ], 22 | "roleConfigGroups": [ 23 | { 24 | "refName": "YARN-1-JOBHISTORY-BASE", 25 | "roleType": "JOBHISTORY", 26 | "configs": [], 27 | "base": true 28 | }, { 29 | "refName": "YARN-1-GATEWAY-BASE", 30 | "roleType": "GATEWAY", 31 | "configs": [ 32 | { 33 | "name": "mapred_submit_replication", 34 | "value": "1" 35 | }, { 36 | "name": "mapred_reduce_tasks", 37 | "value": "6" 38 | } 39 | ], 40 | "base": true 41 | }, { 42 | "refName": "YARN-1-NODEMANAGER-BASE", 43 | "roleType": "NODEMANAGER", 44 | "configs": [ 45 | { 46 | "name": "yarn_nodemanager_local_dirs", 47 | "variable": "YARN_NODEMANAGER_LOCAL_DIRS" 48 | }, { 49 | "name": "yarn_nodemanager_log_dirs", 50 | "variable": "YARN_NODEMANAGER_LOG_DIRS" 51 | } 52 | ], 53 | "base": true 54 | }, { 55 | "refName": "YARN-1-RESOURCEMANAGER-BASE", 56 | "roleType": "RESOURCEMANAGER", 57 | "configs": [], 58 | "base": true 59 | } 60 | ] 61 | } -------------------------------------------------------------------------------- /roles/cdh/templates/zookeeper.j2: -------------------------------------------------------------------------------- 1 | { 2 | "refName": "ZOOKEEPER-1", 3 | "serviceType": "ZOOKEEPER", 4 | "serviceConfigs": [ 5 | { 6 | "name": "zookeeper_datadir_autocreate", 7 | "value": "true" 8 | } 9 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 10 | ,{ 11 | "name" : "enableSecurity", 12 | "value" : "true", 13 | } 14 | {% endif %} 15 | ], 16 | "roleConfigGroups": [ 17 | { 18 | "refName": "ZOOKEEPER-1-SERVER-BASE", 19 | "roleType": "SERVER", 20 | "configs": [], 21 | "base": true 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /roles/cm_agents/tasks/36322.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Temporary workaround for OPSAPS-36322 3 | 4 | - name: Create group 'hadoop' 5 | group: 6 | name: hadoop 7 | 8 | - name: Create user 'flume' 9 | user: 10 | name: flume 11 | home: /var/lib/flume-ng 12 | shell: /sbin/nologin 13 | system: yes 14 | 15 | - name: Create user 'hbase' 16 | user: 17 | name: hbase 18 | home: /var/lib/hbase 19 | shell: /sbin/nologin 20 | system: yes 21 | 22 | - name: Create user 'hdfs' 23 | user: 24 | name: hdfs 25 | groups: 26 | - hadoop 27 | home: /var/lib/hadoop-hdfs 28 | shell: /sbin/nologin 29 | system: yes 30 | 31 | - name: Create user 'hive' 32 | user: 33 | name: hive 34 | home: /var/lib/hive 35 | shell: /sbin/nologin 36 | system: yes 37 | 38 | - name: Create user 'httpfs' 39 | user: 40 | name: httpfs 41 | home: /var/lib/hadoop-httpfs 42 | shell: /sbin/nologin 43 | system: yes 44 | 45 | - name: Create user 'hue' 46 | user: 47 | name: hue 48 | home: /usr/lib/hue 49 | shell: /sbin/nologin 50 | system: yes 51 | 52 | - name: Create user 'impala' 53 | user: 54 | name: impala 55 | groups: 56 | - hive 57 | home: /var/lib/impala 58 | shell: /sbin/nologin 59 | system: yes 60 | 61 | - name: Create user 'kafka' 62 | user: 63 | name: kafka 64 | home: /var/lib/kafka 65 | shell: /sbin/nologin 66 | system: yes 67 | 68 | - name: Create user 'kms' 69 | user: 70 | name: kms 71 | home: /var/lib/hadoop-kms 72 | shell: /sbin/nologin 73 | system: yes 74 | 75 | - name: Create user 'kudu' 76 | user: 77 | name: kudu 78 | home: /var/lib/kudu 79 | shell: /sbin/nologin 80 | system: yes 81 | 82 | - name: Create user 'mapred' 83 | user: 84 | name: mapred 85 | groups: 86 | - hadoop 87 | home: /var/lib/hadoop-mapreduce 88 | shell: /sbin/nologin 89 | system: yes 90 | 91 | - name: Create user 'oozie' 92 | user: 93 | name: oozie 94 | home: /var/lib/oozie 95 | shell: /sbin/nologin 96 | system: yes 97 | 98 | - name: Create user 'sentry' 99 | user: 100 | name: sentry 101 | home: /var/lib/sentry 102 | shell: /sbin/nologin 103 | system: yes 104 | 105 | - name: Create user 'solr' 106 | user: 107 | name: solr 108 | home: /var/lib/solr 109 | shell: /sbin/nologin 110 | system: yes 111 | 112 | - name: Create user 'spark' 113 | user: 114 | name: spark 115 | home: /var/lib/spark 116 | shell: /sbin/nologin 117 | system: yes 118 | 119 | - name: Create user 'sqoop' 120 | user: 121 | name: sqoop 122 | home: /var/lib/sqoop 123 | shell: /sbin/nologin 124 | system: yes 125 | 126 | - name: Create user 'sqoop2' 127 | user: 128 | name: sqoop2 129 | groups: 130 | - sqoop 131 | home: /var/lib/sqoop2 132 | shell: /sbin/nologin 133 | system: yes 134 | when: cluster_version_cdh_major == 5 135 | 136 | - name: Create user 'yarn' 137 | user: 138 | name: yarn 139 | groups: 140 | - hadoop 141 | - spark 142 | home: /var/lib/hadoop-yarn 143 | shell: /sbin/nologin 144 | system: yes 145 | 146 | - name: Create user 'zookeeper' 147 | user: 148 | name: zookeeper 149 | home: /var/lib/zookeeper 150 | shell: /sbin/nologin 151 | system: yes 152 | -------------------------------------------------------------------------------- /roles/cm_agents/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Apply temporary workaround for Cloudera Manager issue OPSAPS-36322 4 | include: 36322.yml 5 | 6 | - name: Install Cloudera Manager Agents 7 | yum: 8 | name: 9 | - cloudera-manager-daemons 10 | - cloudera-manager-agent 11 | update_cache: yes 12 | state: installed 13 | 14 | - name: Configure Cloudera Manager Agent 'server_host' 15 | lineinfile: 16 | dest: /etc/cloudera-scm-agent/config.ini 17 | regexp: "^server_host" 18 | line: "server_host={{ hostvars[scm_hostname]['inventory_hostname'] }}" 19 | 20 | - name: Restart Cloudera Manager Agents 21 | service: 22 | name: cloudera-scm-agent 23 | state: restarted 24 | enabled: yes 25 | -------------------------------------------------------------------------------- /roles/cm_repo/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Add Cloudera Manager yum repository 4 | yum_repository: 5 | name: cloudera-manager 6 | description: Cloudera Manager 7 | baseurl: "{{ scm_repo_url }}" 8 | gpgkey: "{{ scm_repo_gpgkey }}" 9 | gpgcheck: yes 10 | enabled: yes 11 | when: 12 | - ansible_os_family|lower == "redhat" 13 | -------------------------------------------------------------------------------- /roles/epel/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Add EPEL GPG key 4 | rpm_key: 5 | key: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 6 | state: present 7 | 8 | - stat: 9 | path: /etc/yum.repos.d/epel.repo 10 | register: epel_exists 11 | 12 | - name: Add EPEL yum repository 13 | yum_repository: 14 | name: epel 15 | description: Extra Packages for Enterprise Linux $releasever 16 | mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch 17 | failovermethod: priority 18 | gpgcheck: yes 19 | enabled: yes 20 | when: not epel_exists.stat.exists 21 | -------------------------------------------------------------------------------- /roles/java/tasks/install_jce_from_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - lineinfile: 4 | path: "{{ java_home }}/jre/lib/security/java.security" 5 | regexp: '#?crypto.policy=' 6 | line: crypto.policy=unlimited -------------------------------------------------------------------------------- /roles/java/tasks/install_jce_from_zip.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install unzip package 4 | package: 5 | name: 6 | - unzip 7 | state: installed 8 | 9 | - name: Copy JCE policy zip to temp directory 10 | copy: 11 | src: "{{java_jce_location}}" 12 | dest: "{{ tmp_dir }}/jce.zip" 13 | remote_src: "{{java_jce_remote_src}}" 14 | 15 | - name: Extract JCE policy zip 16 | unarchive: 17 | src: "{{ tmp_dir }}/jce.zip" 18 | dest: "{{ tmp_dir }}" 19 | copy: no 20 | 21 | - name: Copy JCE policy jars into correct location 22 | copy: 23 | src: "{{ item }}" 24 | dest: "{{ java_home }}/jre/lib/security/" 25 | backup: yes 26 | remote_src: True 27 | with_fileglob: 28 | - "{{ tmp_dir }}/{{ unarchived_directory }}/*.jar" 29 | 30 | - name: Cleanup tmp files 31 | file: 32 | path: "{{ tmp_dir }}/{{ item }}" 33 | state: absent 34 | with_items: 35 | - jce.zip 36 | - "{{ unarchived_directory }}" 37 | ignore_errors: True 38 | -------------------------------------------------------------------------------- /roles/java/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Copy Java RPM file to temp directory 4 | copy: 5 | src: "{{ java_rpm_location }}" 6 | dest: "{{ tmp_dir }}/jdk.rpm" 7 | remote_src: "{{ java_rpm_remote_src }}" 8 | when: java_installation_strategy == 'rpm' 9 | 10 | - name: Install Java from RPM 11 | package: 12 | name: 13 | - "{{ tmp_dir }}/jdk.rpm" 14 | state: installed 15 | when: java_installation_strategy == 'rpm' 16 | 17 | - name: Install Java from package repository 18 | package: 19 | name: 20 | - "{{ java_package }}" 21 | state: installed 22 | update_cache: yes 23 | when: java_installation_strategy == 'package' 24 | 25 | - name: Add missing symlinks (if installed from Cloudera repo) 26 | block: 27 | - name: Find Java home directory 28 | find: 29 | paths: /usr/java 30 | patterns: 'jdk*-cloudera' 31 | file_type: directory 32 | recurse: no 33 | register: cloudera_jdk_home 34 | - name: Create alternatives symlink for java 35 | alternatives: 36 | name: java 37 | link: /usr/bin/java 38 | path: "{{ cloudera_jdk_home.files[0].path}}/bin/java" 39 | when: cloudera_jdk_home.matched 40 | - name: Create default symlink for Java home directory 41 | file: 42 | src: "{{ cloudera_jdk_home.files[0].path}}" 43 | dest: /usr/java/default 44 | state: link 45 | when: cloudera_jdk_home.matched 46 | when: java_installation_strategy != 'none' 47 | 48 | - name: Capture installed Java provider 49 | raw: /usr/bin/java -version 2>&1 | egrep -o 'Java\(TM\)|OpenJDK' | sed 's/Java(TM)/Oracle/' | tr '[A-Z]' '[a-z]' | head -1 50 | register: provider 51 | when: java_installation_strategy != 'none' 52 | 53 | - name: Capture installed Java version 54 | raw: /usr/bin/java -version 2>&1 | grep version | tr -d '"' | tr "_" " " | awk '{print $3"\n"$4}' 55 | register: version 56 | when: java_installation_strategy != 'none' 57 | 58 | - set_fact: 59 | installed_jdk_provider: "{{ provider.stdout_lines[0] }}" 60 | installed_jdk_version: "{{ version.stdout_lines[0] }}" 61 | installed_jdk_update: "{{ version.stdout_lines[1] }}" 62 | when: java_installation_strategy != 'none' 63 | 64 | - name: Enable Unlimited Strength Policy (Oracle JDK7) 65 | include_tasks: install_jce_from_zip.yml 66 | vars: 67 | java_home: /usr/java/default 68 | unarchived_directory: UnlimitedJCEPolicy 69 | when: java_installation_strategy != 'none' and installed_jdk_provider == 'oracle' and installed_jdk_version == '1.7.0' 70 | 71 | - name: Enable Unlimited Strength Policy (Oracle JDK8 before u151) 72 | include_tasks: install_jce_from_zip.yml 73 | vars: 74 | java_home: /usr/java/default 75 | unarchived_directory: UnlimitedJCEPolicyJDK8 76 | when: java_installation_strategy != 'none' and installed_jdk_provider == 'oracle' and installed_jdk_version == '1.8.0' and installed_jdk_update|int < 151 77 | 78 | - name: Enable Unlimited Strength Policy (Oracle JDK8 after u151) 79 | include_tasks: install_jce_from_config.yml 80 | vars: 81 | java_home: /usr/java/default 82 | when: java_installation_strategy != 'none' and installed_jdk_provider == 'oracle' and installed_jdk_version == '1.8.0' and installed_jdk_update|int >= 151 83 | 84 | - name: Enable Unlimited Strength Policy (OpenJDK) 85 | include_tasks: install_jce_from_config.yml 86 | vars: 87 | java_home: /usr/lib/jvm 88 | when: java_installation_strategy != 'none' and installed_jdk_provider == 'openjdk' and installed_jdk_version is not match("11.*") 89 | 90 | - name: Apply workaround for Kerberos issues introduced in OpenJDK 1.8u242 (JDK-8215032) 91 | lineinfile: 92 | path: /usr/lib/jvm/jre/lib/security/java.security 93 | regexp: '^sun.security.krb5.disableReferrals' 94 | line: sun.security.krb5.disableReferrals=true 95 | when: java_installation_strategy != 'none' and installed_jdk_provider == 'openjdk' and installed_jdk_version == '1.8.0' and installed_jdk_update|int >= 242 96 | 97 | -------------------------------------------------------------------------------- /roles/krb5/client/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - { role: krb5/common } -------------------------------------------------------------------------------- /roles/krb5/client/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install krb5 3 | yum: 4 | name: 5 | - krb5-libs 6 | - krb5-workstation 7 | state: present 8 | 9 | - name: copy krb5.conf 10 | template: 11 | src: krb5.conf.j2 12 | dest: /etc/krb5.conf 13 | mode: 0644 14 | owner: root 15 | group: root 16 | backup: yes 17 | force: yes 18 | -------------------------------------------------------------------------------- /roles/krb5/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | krb5_realm: CLOUDERA.LOCAL 3 | krb5_kdc_type: MIT KDC 4 | krb5_kdc_host: "{{ hostvars[groups['krb5_server'][0]]['inventory_hostname'] }}" 5 | krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" 6 | krb5_kdc_admin_passwd: changeme 7 | krb5_kdc_master_passwd: changeme 8 | krb5_enc_types: aes256-cts 9 | krb5_cm_managed_krb5_conf: false 10 | -------------------------------------------------------------------------------- /roles/krb5/server/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - { role: krb5/common } -------------------------------------------------------------------------------- /roles/krb5/server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install KRB5 server 4 | yum: 5 | name: 6 | - krb5-libs 7 | - krb5-server 8 | state: installed 9 | 10 | - name: Set kdc.conf 11 | template: 12 | src: kdc.conf.j2 13 | dest: /var/kerberos/krb5kdc/kdc.conf 14 | backup: yes 15 | 16 | - name: Set krb5.conf 17 | template: 18 | src: krb5.conf.j2 19 | dest: /etc/krb5.conf 20 | backup: yes 21 | 22 | - name: Create KDC database 23 | command: "/usr/sbin/kdb5_util create -s -P {{ krb5_kdc_master_passwd }}" 24 | args: 25 | creates: /var/kerberos/krb5kdc/principal 26 | 27 | - name: Set kadm5.acl 28 | template: 29 | src: kadm5.acl.j2 30 | dest: /var/kerberos/krb5kdc/kadm5.acl 31 | backup: yes 32 | 33 | - name: Create Cloudera Manager admin principal 34 | command: /usr/sbin/kadmin.local -q "addprinc -pw {{ krb5_kdc_admin_passwd }} {{ krb5_kdc_admin_user }}" 35 | 36 | - name: Start Kerberos 37 | service: 38 | name: "{{ item }}" 39 | state: restarted 40 | enabled: yes 41 | with_items: 42 | - krb5kdc 43 | - kadmin 44 | -------------------------------------------------------------------------------- /roles/krb5/server/templates/kadm5.acl.j2: -------------------------------------------------------------------------------- 1 | */admin@{{ krb5_realm|upper }} * -------------------------------------------------------------------------------- /roles/krb5/server/templates/kdc.conf.j2: -------------------------------------------------------------------------------- 1 | [kdcdefaults] 2 | kdc_ports = 88 3 | kdc_tcp_ports = 88 4 | 5 | [realms] 6 | {{ krb5_realm|upper }} = { 7 | max_renewable_life = 7d 8 | master_key_type = aes256-cts 9 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 10 | dict_file = /usr/share/dict/words 11 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 12 | supported_enctypes = {{ krb5_enc_types }} 13 | } -------------------------------------------------------------------------------- /roles/mariadb/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Handlers for MariaDB. 3 | 4 | - name: restart mariadb 5 | service: 6 | name: mariadb 7 | state: restarted 8 | -------------------------------------------------------------------------------- /roles/mariadb/tasks/databases.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Create databases 4 | mysql_db: 5 | login_user: root 6 | login_password: "{{ mysql_root_password }}" 7 | name: "{{ item.value.name }}" 8 | state: present 9 | encoding: utf8 10 | with_dict: "{{ databases }}" 11 | 12 | - name: Create database users 13 | mysql_user: 14 | login_user: root 15 | login_password: "{{ mysql_root_password }}" 16 | name: "{{ item.value.user }}" 17 | password: "{{ item.value.pass }}" 18 | update_password: always 19 | priv: "{{ item.value.name }}.*:ALL" 20 | host: "%" 21 | state: present 22 | with_dict: "{{ databases }}" 23 | -------------------------------------------------------------------------------- /roles/mariadb/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install MariaDB packages 4 | yum: 5 | name: 6 | - mariadb-server 7 | - MySQL-python 8 | state: installed 9 | 10 | - name: Create MariaDB configuration file 11 | template: 12 | src: my.cnf.j2 13 | dest: /etc/my.cnf 14 | notify: 15 | - restart mariadb 16 | 17 | - name: Create MariaDB log file 18 | file: 19 | path: "{{ mysql_log }}" 20 | state: touch 21 | owner: mysql 22 | group: mysql 23 | mode: 0775 24 | 25 | - name: Create MariaDB PID directory 26 | file: 27 | path: "{{ mysql_pid_dir }}" 28 | state: directory 29 | owner: mysql 30 | group: mysql 31 | mode: 0775 32 | 33 | - name: Start MariaDB Service 34 | service: 35 | name: mariadb 36 | state: started 37 | enabled: yes 38 | 39 | - include: mysql_secure_installation.yml 40 | - include: databases.yml 41 | 42 | 43 | -------------------------------------------------------------------------------- /roles/mariadb/tasks/mysql_secure_installation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Set root password 4 | mysql_user: 5 | name: root 6 | password: "{{ mysql_root_password }}" 7 | state: present 8 | ignore_errors: True 9 | 10 | - name: Remove anonymous users 11 | command: 'mysql -uroot -p{{ mysql_root_password }} -ne "{{ item }}"' 12 | with_items: 13 | - DELETE FROM mysql.user WHERE User='' 14 | 15 | - name: Disable remote login for root 16 | command: 'mysql -uroot -p{{ mysql_root_password }} -ne "{{ item }}"' 17 | with_items: 18 | - DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('127.0.0.1', '::1', 'localhost') 19 | 20 | - name: Remove the test database 21 | mysql_db: 22 | login_user: root 23 | login_password: "{{ mysql_root_password }}" 24 | db: test 25 | state: absent 26 | 27 | - name: Reload privilege tables 28 | command: 'mysql -uroot -p{{ mysql_root_password }} -ne "{{ item }}"' 29 | with_items: 30 | - FLUSH PRIVILEGES 31 | -------------------------------------------------------------------------------- /roles/mariadb/templates/my.cnf.j2: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | datadir = {{ mysql_datadir }} 3 | socket = {{ mysql_socket }} 4 | port = {{ mysql_port }} 5 | 6 | transaction-isolation = READ-COMMITTED 7 | # Disabling symbolic-links is recommended to prevent assorted security risks 8 | symbolic-links = 0 9 | 10 | key_buffer = 16M 11 | key_buffer_size = 32M 12 | max_allowed_packet = 32M 13 | thread_stack = 256K 14 | thread_cache_size = 64 15 | query_cache_limit = 8M 16 | query_cache_size = 64M 17 | query_cache_type = 1 18 | 19 | max_connections = 550 20 | # expire_logs_days = 10 21 | # max_binlog_size = 100M 22 | 23 | # log_bin should be on a disk with enough free space 24 | # Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your system 25 | # and chown the specified folder to the mysql user. 26 | log_bin = {{ mysql_log_bin }} 27 | binlog_format = mixed 28 | 29 | read_buffer_size = 2M 30 | read_rnd_buffer_size = 16M 31 | sort_buffer_size = 8M 32 | join_buffer_size = 8M 33 | 34 | # InnoDB settings 35 | innodb_file_per_table = 1 36 | innodb_flush_log_at_trx_commit = 2 37 | innodb_log_buffer_size = {{ mysql_innodb_log_buffer_size }} 38 | innodb_buffer_pool_size = {{ mysql_innodb_buffer_pool_size }} 39 | innodb_thread_concurrency = 8 40 | innodb_flush_method = O_DIRECT 41 | innodb_log_file_size = 512M 42 | 43 | [mysqld_safe] 44 | log-error = {{ mysql_log }} 45 | pid-file = {{ mysql_pid_file }} -------------------------------------------------------------------------------- /roles/mysql_connector/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Create /usr/share/java directory 4 | file: 5 | path: /usr/share/java 6 | state: directory 7 | mode: 0755 8 | 9 | - name: Install unzip package 10 | package: 11 | name: 12 | - unzip 13 | state: installed 14 | 15 | - stat: 16 | path: /usr/share/java/mysql-connector-java.jar 17 | register: mysql_connector_java 18 | 19 | - name: Download MySQL Connector/J 20 | get_url: 21 | url: https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.48.zip 22 | dest: "{{ tmp_dir }}/mysql-connector-java.zip" 23 | checksum: "md5:5da24facd99964f296ecde32abcd2384" 24 | mode: 0644 25 | when: not mysql_connector_java.stat.exists 26 | 27 | - name: Extract MySQL Connector/J zip file 28 | unarchive: 29 | src: "{{ tmp_dir }}/mysql-connector-java.zip" 30 | dest: "{{ tmp_dir }}" 31 | copy: no 32 | when: not mysql_connector_java.stat.exists 33 | 34 | - name: Copy MySQL Connector/J jar file to correct location 35 | copy: 36 | src: "{{ tmp_dir }}/mysql-connector-java-5.1.48/mysql-connector-java-5.1.48.jar" 37 | dest: /usr/share/java/mysql-connector-java.jar 38 | mode: 0644 39 | remote_src: yes 40 | when: not mysql_connector_java.stat.exists -------------------------------------------------------------------------------- /roles/realm/join/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install packages 3 | yum: 4 | name: 5 | - adcli 6 | - sssd 7 | - realmd 8 | - oddjob 9 | - oddjob-mkhomedir 10 | - python-pip 11 | - openldap-clients 12 | state: present 13 | 14 | - name: Restart the messagebus service, otherwise the realmd service will not start 15 | command: systemctl restart messagebus 16 | 17 | - name: pexpect is required to handle pwd prompt 18 | pip: name=pexpect 19 | 20 | - name: Copy realmd.conf 21 | template: src=realmd.conf.j2 dest=/etc/realmd.conf mode=0644 owner=root group=root backup=yes force=yes 22 | 23 | - name: Restart the realmd service 24 | command: systemctl restart realmd 25 | 26 | - name: Join using realmd 27 | expect: 28 | command: /bin/bash -c "/usr/sbin/realm join {{ ad_domain }} -U {{ bind_user }} --membership-software=adcli --user-principal=host/{{inventory_hostname}}@{{krb5_realm}}" 29 | responses: 30 | Password for .*: "{{ bind_password }}" 31 | 32 | - name: Copy sssd.conf 33 | template: src=sssd.conf.j2 dest=/etc/sssd/sssd.conf mode=0600 owner=root group=root backup=yes force=yes 34 | 35 | - name: Restart the sssd service 36 | command: systemctl restart sssd 37 | 38 | - name: Copy nscd.conf 39 | template: src=nscd.conf.j2 dest=/etc/nscd.conf mode=0644 owner=root group=root backup=yes force=yes 40 | 41 | - name: Restart the nscd service 42 | command: systemctl restart nscd 43 | -------------------------------------------------------------------------------- /roles/realm/join/templates/nscd.conf.j2: -------------------------------------------------------------------------------- 1 | # max-threads 32 2 | server-user nscd 3 | # stat-user somebody 4 | debug-level 0 5 | # reload-count 5 6 | paranoia no 7 | # restart-interval 3600 8 | 9 | enable-cache passwd no 10 | positive-time-to-live passwd 600 11 | negative-time-to-live passwd 20 12 | suggested-size passwd 211 13 | check-files passwd yes 14 | persistent passwd yes 15 | shared passwd yes 16 | max-db-size passwd 33554432 17 | auto-propagate passwd yes 18 | 19 | enable-cache group no 20 | positive-time-to-live group 3600 21 | negative-time-to-live group 60 22 | suggested-size group 211 23 | check-files group yes 24 | persistent group yes 25 | shared group yes 26 | max-db-size group 33554432 27 | auto-propagate group yes 28 | 29 | enable-cache hosts yes 30 | positive-time-to-live hosts 3600 31 | negative-time-to-live hosts 20 32 | suggested-size hosts 211 33 | check-files hosts yes 34 | persistent hosts yes 35 | shared hosts yes 36 | max-db-size hosts 33554432 37 | 38 | enable-cache services no 39 | positive-time-to-live services 28800 40 | negative-time-to-live services 20 41 | suggested-size services 211 42 | check-files services yes 43 | persistent services yes 44 | shared services yes 45 | max-db-size services 33554432 46 | 47 | enable-cache netgroup no 48 | positive-time-to-live netgroup 28800 49 | negative-time-to-live netgroup 20 50 | suggested-size netgroup 211 51 | check-files netgroup yes 52 | persistent netgroup yes 53 | shared netgroup yes 54 | max-db-size netgroup 33554432 55 | -------------------------------------------------------------------------------- /roles/realm/join/templates/realmd.conf.j2: -------------------------------------------------------------------------------- 1 | [{{ ad_domain }}] 2 | computer-ou = {{ computer_ou }} 3 | # Please note that the computer-name / netbios name cannot exceed 15 characters 4 | computer-name = {{ inventory_hostname_short[-15:]}} 5 | -------------------------------------------------------------------------------- /roles/realm/join/templates/sssd.conf.j2: -------------------------------------------------------------------------------- 1 | [sssd] 2 | 3 | config_file_version = 2 4 | reconnection_retries = 3 5 | sbus_timeout = 30 6 | services = nss, pam 7 | domains = {{ ad_domain }} 8 | 9 | [nss] 10 | 11 | # There are two types of users which are available at the operating system level: service users (yarn, hdfs, etc.) and non-service users. Cloudera recommends that the service users be local users and the non-service users are in LDAP-compliant identity/directory services, such as OpenLDAP or Microsoft Active Directory. 12 | 13 | filter_groups = root,mysql,hadoop,yarn,hdfs,mapred,kms,httpfs,hbase,hive,sentry,spark,solr,sqoop,oozie,hue,flume,impala,llama,postgres,sqoop2,kudu,kafka,accumulo,zookeeper,cloudera-scm,keytrustee 14 | 15 | filter_users = root,mysql,cloudera-scm,zookeeper,yarn,hdfs,mapred,kms,httpfs,hbase,hive,sentry,spark,solr,sqoop,oozie,hue,flume,impala,llama,sqoop2,postgres,kudu,kafka,accumulo,keytrustee 16 | 17 | reconnection_retries = 3 18 | 19 | # entry_cache_timeout = 3600 20 | # entry_negative_timeout = 30 21 | # entry_cache_nowait_percentage = 7 22 | 23 | [pam] 24 | 25 | reconnection_retries = 3 26 | 27 | [domain/{{ ad_domain }}] 28 | 29 | debug_level = 3 30 | ad_enable_gc = false 31 | cache_credentials = true 32 | ldap_id_mapping = true 33 | 34 | # The domain users group is usually the primary group ID. If you correctly override it with override_gid, it should not appear unless you have specifically added the users to that group. If your group filter is correct, it does not search the organization unit where domain users exist and it will be filtered out. 35 | 36 | # When override_gid is enabled, the primary group of all the users in the domain is set to the value of override_gid. You must change the override_gid default value. 37 | 38 | override_gid = {{ override_gid }} 39 | 40 | # The auto_private_groups option was added in SSSD 1.16.1. If this option is set to true, each user’s GID will be changed to match their UID. Do not enable override_gid and auto_private_groups at the same time. 41 | 42 | # auto_private_groups = true 43 | 44 | use_fully_qualified_names = false 45 | full_name_format = %1$s 46 | fallback_homedir = /home/%u 47 | default_shell = /bin/bash 48 | ignore_group_members = true 49 | krb5_realm = {{ krb5_realm }} 50 | ad_domain = {{ ad_domain }} 51 | id_provider = ad 52 | chpass_provider = ad 53 | auth_provider = ad 54 | 55 | # Use the ad_access_filter option to specify groups that can login to the host via SSH. The default setting gives SSH access to members of the hadoop_admins and hadoop_users groups. Use this setting with caution. 56 | 57 | access_provider = ad 58 | 59 | # ad_access_filter = (|(memberOf=CN=hadoop_admins,OU=groups,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com)(memberOf=CN=hadoop_users,OU=groups,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com)) 60 | 61 | # Cloudera recommends that you enable ad_site by specifying an Active Directory site with the ad_site option. When ad_site is enabled, SSSD discovers valid AD servers for the site and updates the server list automatically when the AD servers change. You do not have to hardcode the servers with the ad_server option. 62 | 63 | ad_site = {{ ad_site }} 64 | 65 | case_sensitive = false 66 | enumerate = false 67 | ldap_schema = ad 68 | ldap_user_principal = nosuchattr 69 | ldap_force_upper_case_realm = true 70 | ldap_purge_cache_timeout = 0 71 | ldap_account_expire_policy = ad 72 | 73 | # Setting subdomains_provider to none disallows fetching subdomains explicitly. This can help resolve significant delays if there are multiple AD domains. Note that there is a known issue in versions of RHEL 6. For more information, see https://bugzilla.redhat.com/show_bug.cgi?id=1221358 74 | 75 | # subdomains_provider = none 76 | 77 | # If you know that the users and groups will be limited to a specific subsection of the directory, you can limit lookups with the ldap_group_search_base and ldap_user_search_base parameters. This can give significant performance improvements for large directories. 78 | 79 | ldap_group_search_base = {{ ldap_group_search_base }} 80 | ldap_user_search_base = {{ ldap_user_search_base }} 81 | 82 | # Uncomment the following lines if you are deploying on Azure, which has no reverse DNS, or if your environment has DNS managed by Active Directory. 83 | 84 | # dyndns_update = true 85 | # dyndns_refresh_interval = 43200 86 | # dyndns_update_ptr = true 87 | # dyndns_ttl = 3600 88 | 89 | ldap_use_tokengroups = false 90 | 91 | # Setting krb5_validate to false speeds up the login procedure by preventing TGT verification. 92 | 93 | krb5_validate = false 94 | 95 | # In environments with a large number of users and groups, you must set the idmap range with the ldap_idmap_range_size option. 96 | 97 | # ldap_idmap_range_size = 2000000 98 | 99 | ldap_sasl_authid = host/{{inventory_hostname}}@{{krb5_realm}} 100 | -------------------------------------------------------------------------------- /roles/realm/leave/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Remove hosts from the realm 3 | command: realm leave 4 | -------------------------------------------------------------------------------- /roles/rngd/handlers/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Handlers for rngd. 3 | 4 | - name: restart rngd 5 | service: 6 | name: rngd 7 | state: restarted 8 | -------------------------------------------------------------------------------- /roles/rngd/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: ensure rng-tools is installed 4 | yum: 5 | name: rng-tools 6 | state: installed 7 | update_cache: yes 8 | 9 | - name: Create rngd configuration file 10 | template: 11 | src: rngd.service.j2 12 | dest: /etc/systemd/system/rngd.service 13 | notify: 14 | - restart rngd 15 | 16 | - name: ensure rngd is running 17 | service: 18 | name: rngd 19 | state: started 20 | enabled: yes 21 | -------------------------------------------------------------------------------- /roles/rngd/templates/rngd.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Hardware RNG Entropy Gatherer Daemon 3 | 4 | [Service] 5 | ExecStart=/sbin/rngd -f -r /dev/urandom 6 | 7 | [Install] 8 | WantedBy=multi-user.target 9 | -------------------------------------------------------------------------------- /roles/scm/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Handlers for Cloudera Manager Server. 3 | 4 | - name: restart cloudera-scm-server 5 | service: 6 | name: cloudera-scm-server 7 | state: restarted 8 | notify: 9 | - wait cloudera-scm-server 10 | 11 | - name: wait cloudera-scm-server 12 | wait_for: 13 | host: "{{ hostvars[scm_hostname]['inventory_hostname'] }}" 14 | port: "{{ scm_port }}" 15 | delay: 5 16 | state: started 17 | timeout: 300 18 | -------------------------------------------------------------------------------- /roles/scm/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - { role: krb5/common } -------------------------------------------------------------------------------- /roles/scm/tasks/cms.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Wait for agents to send heartbeats in case SCM has just been restarted 4 | # Adding CMS will fail if host details haven't been reported in 5 | - name: Wait for agent heartbeats 6 | pause: 7 | seconds: 30 8 | 9 | - set_fact: 10 | cms_config_json: "{{ lookup('template', 'cms_base.j2', convert_data=False) }}" 11 | 12 | # https://cloudera.github.io/cm_api/apidocs/v12/path__cm_service.html#PUT 13 | - name: Setup the Cloudera Management Services (CMS) 14 | uri: 15 | url: "{{ cm_api_url }}/cm/service" 16 | method: PUT 17 | body_format: json 18 | body: "{{ cms_config_json|to_json }}" 19 | status_code: 200,400 20 | force_basic_auth: yes 21 | user: "{{ scm_default_user }}" 22 | password: "{{ scm_default_pass }}" 23 | return_content: yes 24 | register: cms_resp 25 | failed_when: 26 | - "'MGMT' not in cms_resp.content" 27 | - "'CMS instance already exists' not in cms_resp.content" 28 | 29 | - debug: 30 | var: cms_resp 31 | verbosity: 1 32 | 33 | # https://cloudera.github.io/cm_api/apidocs/v12/path__cm_service_commands_start.html 34 | - name: Start Cloudera Management Services (CMS) 35 | uri: 36 | url: "{{ cm_api_url }}/cm/service/commands/start" 37 | method: POST 38 | status_code: 200 39 | force_basic_auth: yes 40 | user: "{{ scm_default_user }}" 41 | password: "{{ scm_default_pass }}" 42 | return_content: yes 43 | register: start_resp 44 | failed_when: "'startTime' not in start_resp.content" 45 | 46 | - debug: 47 | var: start_resp 48 | verbosity: 1 49 | -------------------------------------------------------------------------------- /roles/scm/tasks/license.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Check license file exists 4 | stat: 5 | path: "{{ scm_license_file }}" 6 | register: file 7 | 8 | - debug: 9 | msg: "License file '{{ scm_license_file }}' exists = {{ file.stat.exists }}" 10 | verbosity: 1 11 | 12 | # https://cloudera.github.io/cm_api/apidocs/v12/path__cm_license.html 13 | - name: Upload license file to Cloudera Manager 14 | shell: > 15 | curl 16 | -u {{ scm_default_user }}:{{ scm_default_pass }} 17 | -X POST -H 'Content-Type:multipart/form-data' 18 | -F license=@{{ scm_license_file }} 19 | {{ cm_api_url }}/cm/license 20 | args: 21 | warn: False 22 | register: resp 23 | failed_when: "'owner' not in resp.stdout" 24 | notify: 25 | - restart cloudera-scm-server 26 | when: file.stat.exists 27 | 28 | - debug: 29 | var: resp 30 | verbosity: 1 31 | 32 | # https://cloudera.github.io/cm_api/apidocs/v12/path__cm_trial_begin.html 33 | - name: Begin Cloudera Manager trial license 34 | uri: 35 | url: "{{ cm_api_url }}/cm/trial/begin" 36 | method: POST 37 | # Supported Status codes: 38 | # 204: when posting trial/begin (the 1.time) 39 | # 400: when posting trial/begin after trial has begun 40 | status_code: 200,204,400 41 | force_basic_auth: yes 42 | user: "{{ scm_default_user }}" 43 | password: "{{ scm_default_pass }}" 44 | notify: 45 | - restart cloudera-scm-server 46 | when: not file.stat.exists 47 | 48 | - meta: flush_handlers 49 | 50 | 51 | -------------------------------------------------------------------------------- /roles/scm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - include_vars: ../../../group_vars/db_server.yml 4 | 5 | - name: Install the Cloudera Manager Server Packages 6 | yum: 7 | name: 8 | - cloudera-manager-daemons 9 | - cloudera-manager-server 10 | - cloudera-manager-agent 11 | - openldap-clients 12 | state: installed 13 | 14 | - name: Prepare Cloudera Manager Server External Database 15 | command: | 16 | {{ scm_prepare_database_script_path }} -f 17 | --host {{ hostvars[db_hostname]['inventory_hostname'] }} 18 | {{ database_type }} {{ databases.scm.name }} {{ databases.scm.user }} {{ databases.scm.pass }} 19 | changed_when: False 20 | 21 | - name: Create CSD directory 22 | file: 23 | path: /opt/cloudera/csd 24 | state: directory 25 | owner: cloudera-scm 26 | group: cloudera-scm 27 | mode: 0755 28 | 29 | - name: Download CSDs 30 | get_url: 31 | url: "{{ item }}" 32 | dest: /opt/cloudera/csd 33 | mode: 0644 34 | with_items: "{{ scm_csds }}" 35 | when: scm_csds is defined 36 | 37 | - name: Start the Cloudera Manager Server 38 | service: 39 | name: "{{ item }}" 40 | state: started 41 | enabled: yes 42 | notify: 43 | - wait cloudera-scm-server 44 | with_items: 45 | - cloudera-scm-server 46 | - cloudera-scm-agent 47 | 48 | # Trigger handler to wait for SCM to startup 49 | - meta: flush_handlers 50 | 51 | - name: Get Cloudera Manager API version 52 | uri: 53 | url: http://{{ hostvars[scm_hostname]['inventory_hostname'] }}:{{ scm_port }}/api/version 54 | method: GET 55 | status_code: 200 56 | user: "{{ scm_default_user }}" 57 | password: "{{ scm_default_pass }}" 58 | force_basic_auth: yes 59 | return_content: yes 60 | register: result 61 | 62 | # Set base CM API URL 63 | - set_fact: 64 | cm_api_url: "http://{{ hostvars[scm_hostname]['inventory_hostname'] }}:{{ scm_port }}/api/{{ result.content }}" 65 | 66 | - debug: 67 | var: cm_api_url 68 | verbosity: 1 69 | 70 | - name: Get the host identifiers and names from Cloudera Manager 71 | uri: 72 | url: "{{ cm_api_url }}/hosts" 73 | method: GET 74 | status_code: 200 75 | user: "{{ scm_default_user }}" 76 | password: "{{ scm_default_pass }}" 77 | force_basic_auth: yes 78 | return_content: yes 79 | register: scm_host_list 80 | 81 | - name: Extract the host identifiers and names into facts 82 | set_fact: 83 | scm_hosts: "{{ lookup('template', 'scm_host_list.j2') | from_yaml }}" 84 | 85 | - name: Print the extracted host identifiers and names 86 | debug: 87 | var: scm_hosts 88 | verbosity: 2 89 | 90 | - include: license.yml 91 | - include: scm.yml 92 | - include: cms.yml 93 | -------------------------------------------------------------------------------- /roles/scm/tasks/scm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # https://cloudera.github.io/cm_api/apidocs/v13/path__cm_config.html 4 | - name: Update Cloudera Manager settings 5 | uri: 6 | url: "{{ cm_api_url }}/cm/config" 7 | method: PUT 8 | body_format: json 9 | body: "{{ lookup('template', 'scm.j2', convert_data=False) }}" 10 | status_code: 200 11 | force_basic_auth: yes 12 | user: "{{ scm_default_user }}" 13 | password: "{{ scm_default_pass }}" 14 | return_content: yes 15 | register: scm_resp 16 | 17 | - debug: 18 | var: scm_resp 19 | verbosity: 1 20 | 21 | # https://cloudera.github.io/cm_api/apidocs/v13/path__cm_commands_importAdminCredentials.html 22 | - name: Import KDC admin credentials 23 | uri: 24 | url: "{{ cm_api_url }}/cm/commands/importAdminCredentials?username={{ krb5_kdc_admin_user }}&password={{ krb5_kdc_admin_passwd }}" 25 | method: POST 26 | status_code: 200 27 | force_basic_auth: yes 28 | user: "{{ scm_default_user }}" 29 | password: "{{ scm_default_pass }}" 30 | return_content: yes 31 | register: response 32 | when: krb5_kdc_type != 'none' 33 | 34 | - name: Wait for KDC admin credentials import to complete 35 | uri: 36 | url: "{{ cm_api_url }}/commands/{{ response.json.id }}" 37 | body_format: json 38 | force_basic_auth: yes 39 | user: "{{ scm_default_user }}" 40 | password: "{{ scm_default_pass }}" 41 | return_content: yes 42 | register: result 43 | until: not result.json.active 44 | retries: 30 45 | delay: 10 46 | when: krb5_kdc_type != 'none' 47 | failed_when: result.json.success is defined and not result.json.success 48 | -------------------------------------------------------------------------------- /roles/scm/templates/cms_base.j2: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mgmt", 3 | "type": "MGMT", 4 | "roles": [ 5 | { 6 | "name": "mgmt-SERVICEMONITOR", 7 | "type": "SERVICEMONITOR", 8 | "hostRef": { 9 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 10 | } 11 | }, { 12 | "name": "mgmt-ACTIVITYMONITOR", 13 | "type": "ACTIVITYMONITOR", 14 | "roleConfigGroupRef": { 15 | "roleConfigGroupName": "mgmt-ACTIVITYMONITOR-BASE" 16 | }, 17 | "hostRef": { 18 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 19 | } 20 | }, { 21 | "name": "mgmt-HOSTMONITOR", 22 | "type": "HOSTMONITOR", 23 | "hostRef": { 24 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 25 | } 26 | }, { 27 | "name": "mgmt-REPORTSMANAGER", 28 | "type": "REPORTSMANAGER", 29 | "roleConfigGroupRef": { 30 | "roleConfigGroupName": "mgmt-REPORTSMANAGER-BASE" 31 | }, 32 | "hostRef": { 33 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 34 | } 35 | }, { 36 | "name": "mgmt-EVENTSERVER", 37 | "type": "EVENTSERVER", 38 | "hostRef": { 39 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 40 | } 41 | }, { 42 | "name": "mgmt-ALERTPUBLISHER", 43 | "type": "ALERTPUBLISHER", 44 | "hostRef": { 45 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 46 | } 47 | }, { 48 | "name": "mgmt-NAVIGATOR", 49 | "type": "NAVIGATOR", 50 | "roleConfigGroupRef": { 51 | "roleConfigGroupName": "mgmt-NAVIGATOR-BASE" 52 | }, 53 | "hostRef": { 54 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 55 | } 56 | }, { 57 | "name": "mgmt-NAVIGATORMETASERVER", 58 | "type": "NAVIGATORMETASERVER", 59 | "roleConfigGroupRef": { 60 | "roleConfigGroupName": "mgmt-NAVIGATORMETASERVER-BASE" 61 | }, 62 | "hostRef": { 63 | "hostId": "{{ scm_hosts['ids'][scm_hostname] }}" 64 | } 65 | } 66 | ], 67 | "roleConfigGroups": [ 68 | { 69 | "name": "mgmt-ACTIVITYMONITOR-BASE", 70 | "displayName": "Activity Monitor Default Group", 71 | "roleType": "ACTIVITYMONITOR", 72 | "base": true, 73 | "serviceRef": { 74 | "serviceName": "mgmt" 75 | }, 76 | "config": { 77 | "items": [ 78 | { 79 | "name": "firehose_database_host", 80 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 81 | }, { 82 | "name": "firehose_database_name", 83 | "value": "{{ databases.amon.name }}" 84 | }, { 85 | "name": "firehose_database_password", 86 | "value": "{{ databases.amon.pass }}" 87 | }, { 88 | "name": "firehose_database_type", 89 | "value": "{{ databases.amon.type }}" 90 | }, { 91 | "name": "firehose_database_user", 92 | "value": "{{ databases.amon.user }}" 93 | } 94 | ] 95 | } 96 | }, { 97 | "name": "mgmt-REPORTSMANAGER-BASE", 98 | "displayName": "Reports Manager Default Group", 99 | "roleType": "REPORTSMANAGER", 100 | "base": true, 101 | "serviceRef": { 102 | "serviceName": "mgmt" 103 | }, 104 | "config": { 105 | "items": [ 106 | { 107 | "name": "headlamp_database_host", 108 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 109 | }, { 110 | "name": "headlamp_database_name", 111 | "value": "{{ databases.rman.name }}" 112 | }, { 113 | "name": "headlamp_database_password", 114 | "value": "{{ databases.rman.pass }}" 115 | }, { 116 | "name": "headlamp_database_type", 117 | "value": "{{ databases.rman.type }}" 118 | }, { 119 | "name": "headlamp_database_user", 120 | "value": "{{ databases.rman.user }}" 121 | } 122 | ] 123 | } 124 | }, { 125 | "name": "mgmt-NAVIGATOR-BASE", 126 | "displayName": "Navigator Audit Server Default Group", 127 | "roleType": "NAVIGATOR", 128 | "base": true, 129 | "serviceRef": { 130 | "serviceName": "mgmt" 131 | }, 132 | "config": { 133 | "items": [ 134 | { 135 | "name": "navigator_database_host", 136 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 137 | }, { 138 | "name": "navigator_database_name", 139 | "value": "{{ databases.nav.name }}" 140 | }, { 141 | "name": "navigator_database_password", 142 | "value": "{{ databases.nav.pass }}" 143 | }, { 144 | "name": "navigator_database_type", 145 | "value": "{{ databases.nav.type }}" 146 | }, { 147 | "name": "navigator_database_user", 148 | "value": "{{ databases.nav.user }}" 149 | } 150 | ] 151 | } 152 | }, { 153 | "name": "mgmt-NAVIGATORMETASERVER-BASE", 154 | "displayName": "Navigator Metadata Server Default Group", 155 | "roleType": "NAVIGATORMETASERVER", 156 | "base": true, 157 | "serviceRef": { 158 | "serviceName": "mgmt" 159 | }, 160 | "config": { 161 | "items": [ 162 | { 163 | "name": "nav_metaserver_database_host", 164 | "value": "{{ hostvars[db_hostname]['inventory_hostname'] }}" 165 | }, { 166 | "name": "nav_metaserver_database_name", 167 | "value": "{{ databases.navms.name }}" 168 | }, { 169 | "name": "nav_metaserver_database_password", 170 | "value": "{{ databases.navms.pass }}" 171 | }, { 172 | "name": "nav_metaserver_database_type", 173 | "value": "{{ databases.navms.type }}" 174 | }, { 175 | "name": "nav_metaserver_database_user", 176 | "value": "{{ databases.navms.user }}" 177 | } 178 | ] 179 | } 180 | } 181 | ] 182 | } 183 | -------------------------------------------------------------------------------- /roles/scm/templates/scm.j2: -------------------------------------------------------------------------------- 1 | { 2 | "items" : [ { 3 | {% if (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') %} 4 | "name" : "KDC_HOST", 5 | "value" : "{{ krb5_kdc_host }}" 6 | }, { 7 | "name" : "KDC_TYPE", 8 | "value" : "{{ krb5_kdc_type }}" 9 | }, { 10 | "name" : "SECURITY_REALM", 11 | "value" : "{{ krb5_realm | upper }}" 12 | }, { 13 | "name" : "KRB_MANAGE_KRB5_CONF", 14 | "value" : {{ krb5_cm_managed_krb5_conf | lower }} 15 | {% if krb5_enc_types is defined %} 16 | }, { 17 | "name" : "KRB_ENC_TYPES", 18 | "value" : "{{ krb5_enc_types }}" 19 | {% endif %} 20 | {% if krb5_kdc_type == 'Active Directory' %} 21 | {% if krb5_ad_delete_on_regenerate is defined %} 22 | }, { 23 | "name" : "AD_DELETE_ON_REGENERATE", 24 | "value" : {{ krb5_ad_delete_on_regenerate | lower }} 25 | {% endif %} 26 | {% if krb5_ad_suffix is defined %} 27 | }, { 28 | "name" : "AD_KDC_DOMAIN", 29 | "value" : "{{ krb5_ad_suffix }}" 30 | {% endif %} 31 | {% endif %} 32 | }, { 33 | {% endif %} 34 | "name" : "REMOTE_PARCEL_REPO_URLS", 35 | "value" : {% set repo_j = joiner(",") %} 36 | "{% for repo in scm_parcel_repositories %}{{ repo_j() }}{{ repo }}{% endfor %}" 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /roles/scm/templates/scm_host_list.j2: -------------------------------------------------------------------------------- 1 | {%- set scm_hosts = { "ids" : {}, "names" : {} } -%} 2 | {%- for host, vars in hostvars.items() -%} 3 | {%- for scm_host in scm_host_list['json']['items'] -%} 4 | {%- set found_host = False -%} 5 | {%- if scm_host.hostname == vars.inventory_hostname or scm_host.ipAddress == vars.inventory_hostname -%} 6 | {%- set found_host = True -%} 7 | {%- elif alternative_ip|default('private_ip') in vars and scm_host.ipAddress == vars[alternative_ip|default('private_ip')] -%} 8 | {%- set found_host = True -%} 9 | {%- endif -%} 10 | {%- if found_host -%} 11 | {%- set x = scm_hosts.ids.__setitem__(host, scm_host.hostId) -%} 12 | {%- set x = scm_hosts.names.__setitem__(host, scm_host.hostname) -%} 13 | {%- endif -%} 14 | {%- endfor -%} 15 | {%- endfor -%} 16 | {{ scm_hosts|to_yaml }} 17 | -------------------------------------------------------------------------------- /site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Cloudera playbook 3 | 4 | - name: Configure Cloudera Manager Repository 5 | hosts: cdh_servers 6 | roles: 7 | - cm_repo 8 | tags: cm_repo 9 | 10 | - name: Install rngd 11 | hosts: cdh_servers 12 | roles: 13 | - rngd 14 | tags: rngd 15 | 16 | - name: Install Java 17 | hosts: cdh_servers 18 | roles: 19 | - java 20 | tags: java 21 | 22 | - name: Install MariaDB and create databases 23 | hosts: db_server 24 | roles: 25 | - mariadb 26 | tags: 27 | - mysql 28 | - mysql_server 29 | 30 | - name: Install MySQL Connector 31 | hosts: cdh_servers 32 | roles: 33 | - mysql_connector 34 | tags: 35 | - mysql 36 | - mysql_client 37 | 38 | - name: Install MIT KDC Server 39 | hosts: krb5_server 40 | roles: 41 | - krb5/server 42 | tags: krb5 43 | 44 | - name: Install MIT KDC Client 45 | hosts: all 46 | roles: 47 | - { role: krb5/client, when: (krb5_kdc_type is defined) and (krb5_kdc_type != 'none') } 48 | tags: 49 | - krb5 50 | - krb5_client 51 | 52 | - name: Configure EPEL Repository 53 | hosts: scm_server 54 | roles: 55 | - epel 56 | tags: epel 57 | 58 | - name: Install Cloudera Manager Agents 59 | hosts: cdh_servers 60 | roles: 61 | - cm_agents 62 | tags: cm_agents 63 | 64 | - name: Install Cloudera Manager Server 65 | hosts: scm_server 66 | roles: 67 | - scm 68 | tags: cluster_template 69 | 70 | - name: Install CDH 71 | hosts: scm_server 72 | roles: 73 | - cdh 74 | tags: cluster_template 75 | 76 | -------------------------------------------------------------------------------- /templates/krb5.conf.j2: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = {{ krb5_realm|upper }} 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 1d 6 | renew_lifetime = 7d 7 | forwardable = true 8 | default_tgs_enctypes = {{ krb5_enc_types }} 9 | default_tkt_enctypes = {{ krb5_enc_types }} 10 | permitted_enctypes = {{ krb5_enc_types }} 11 | udp_preference_limit = 1 12 | kdc_timeout = 3000 13 | 14 | [realms] 15 | {{ krb5_realm|upper }} = { 16 | kdc = {{ krb5_kdc_host }} 17 | admin_server = {{ krb5_kdc_host }} 18 | } 19 | 20 | [domain_realm] 21 | .{{ ansible_domain }} = {{ krb5_realm|upper }} 22 | {{ ansible_domain }} = {{ krb5_realm|upper }} 23 | --------------------------------------------------------------------------------