├── .github └── workflows │ ├── build-cloudera-stack-release.yml │ └── build-cloudera-stack.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.sh ├── compute.tf ├── data.tf ├── iam.tf ├── images ├── 01 - manager.png ├── 02 - home.png ├── RM_variables.png ├── cloudera_logo.png └── deployment_architecture.png ├── modules ├── bastion │ ├── main.tf │ └── variables.tf ├── master │ ├── main.tf │ └── variables.tf ├── network │ ├── data.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── utility │ ├── data.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── worker │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── network.tf ├── outputs.tf ├── schema.yaml ├── scripts ├── README.md ├── boot.sh ├── cloudera_manager_boot.sh ├── cms_mysql.sh ├── cms_postgres.sh ├── deploy_on_oci.py ├── exportBDC.py └── my.cnf ├── variables.tf └── versions.tf /.github/workflows/build-cloudera-stack-release.yml: -------------------------------------------------------------------------------- 1 | name: 'cloudera-stack-release' 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | terraform_generate_packages: 10 | name: Generate Stack Packages 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Terraform Init 15 | uses: hashicorp/terraform-github-actions@master 16 | with: 17 | tf_actions_version: 0.12.17 18 | tf_actions_subcommand: 'init' 19 | tf_actions_working_dir: terraform 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | - name: 'Terraform Validate' 23 | uses: hashicorp/terraform-github-actions@master 24 | with: 25 | tf_actions_version: 0.12.17 26 | tf_actions_subcommand: 'validate' 27 | tf_actions_working_dir: terraform 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | - name: Build artifacts 31 | run: | 32 | /bin/bash -e build.sh 33 | - name: Upload BYOL stack 34 | uses: actions/upload-release-asset@v1.0.1 35 | with: 36 | upload_url: ${{ github.event.release.upload_url }} 37 | asset_path: ./build/cloudera-stack-byol.zip 38 | asset_name: cloudera-stack-${{ github.event.release.tag_name }}-byol.zip 39 | asset_content_type: application/zip 40 | env: 41 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 42 | -------------------------------------------------------------------------------- /.github/workflows/build-cloudera-stack.yml: -------------------------------------------------------------------------------- 1 | name: 'cloudera-stack' 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | terraform_generate_packages: 7 | name: Generate Stack Packages 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Terraform Init 12 | uses: hashicorp/terraform-github-actions@master 13 | with: 14 | tf_actions_version: 0.12.17 15 | tf_actions_subcommand: 'init' 16 | tf_actions_working_dir: terraform 17 | env: 18 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 19 | - name: 'Terraform Validate' 20 | uses: hashicorp/terraform-github-actions@master 21 | with: 22 | tf_actions_version: 0.12.17 23 | tf_actions_subcommand: 'validate' 24 | tf_actions_working_dir: terraform 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | - name: Build artifacts 28 | run: | 29 | /bin/bash -e build.sh 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # .tfvars files 9 | *.tfvars 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018-2019 Cloudera 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |
4 |

Cloudera on Oracle Cloud Infrastructure

5 |
6 |

7 | 8 | ![cloudera-stack](https://github.com/oracle-quickstart/oci-cloudera/workflows/cloudera-stack/badge.svg) 9 | 10 | This is a Terraform module that deploys [Cloudera Data Platform (CDP) Data Center](https://www.cloudera.com/products/cloudera-data-platform.html) on [Oracle Cloud Infrastructure (OCI)](https://cloud.oracle.com/en_US/cloud-infrastructure). It is developed jointly by Oracle and Cloudera. 11 | 12 | ## Deployment Information 13 | The following table shows Recommended and Minimum supported OCI shapes for each cluster role: 14 | 15 | | | Worker Nodes | Bastion Instance | Utility and Master Instances | 16 | |-------------|----------------|------------------|------------------------------| 17 | | Recommended | BM.DenseIO2.52 | VM.Standard2.4 | VM.Standard2.16 | 18 | | Minimum | VM.Standard2.8 | VM.Standard2.1 | VM.Standard2.8 | 19 | 20 | ## Resource Manager Deployment 21 | This Quick Start uses [OCI Resource Manager](https://docs.cloud.oracle.com/iaas/Content/ResourceManager/Concepts/resourcemanager.htm) to make deployment quite easy. 22 | 23 | Simply click this button to deploy to OCI. 24 | 25 | [![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://console.us-ashburn-1.oraclecloud.com/resourcemanager/stacks/create?region=home&zipUrl=https://github.com/oracle-quickstart/oci-cloudera/archive/v3.3.7.zip) 26 | 27 | This template uses Terraform v0.12, and has support to target existing VCN/Subnets for cluster deployment. To engage this functionality, just use the Schema menu system to select an existing VCN target, then select appropriate Subnets for each cluster host type. 28 | 29 | If you deploy Cloudera Manager to a private subnet, you will require a VPN or SSH Tunnel through an edge node to access cluster management. 30 | 31 | Once the deployment is complete you can access Cloudera manager at `http://:7180/cmf/login`. 32 | 33 | Cluster Provisioning is executed on the Utility host using CloudInit. That activity is logged in /var/log/cloudera-OCI-initialize.log. This log file can be used to triage cluster setup issues. 34 | 35 | ![](images/01%20-%20manager.png) 36 | 37 | The default username is `cm_admin` and the default password is `changeme`. You should see a cluster up and running like this: 38 | 39 | ![](images/02%20-%20home.png) 40 | 41 | If upon login you are presenetd with a licensing prompt, please wait, do not interact, and allow additional time for the automated cluster provisioning process to complete. Refresh the page after a few minutes to check on deployment. 42 | 43 | ## Python Deployment using cm_client 44 | The deployment script `deploy_on_oci.py` uses cm_client against Cloudera Manager API v31. This script can be customized before execution. Reference the header section in the script, the following parameters are passed at deployment time either manually or via ORM schema: 45 | 46 | admin_user_name 47 | admin_password 48 | 49 | When using ORM schema, these values are put into Utility instance metadata. It is highly encouraged to modify the admin password in Cloudera Manager after deployment is complete. 50 | 51 | In addition, advanced customization of the cluster deployment can be done by modification of the following functions: 52 | 53 | setup_mgmt_rcg 54 | update_cluster_rcg_configuration 55 | 56 | This requires some knowledge of Python and Cloudera configuration - modify at your own risk. These functions contain Cloudera specific tuning parameters as well as host mapping for roles. 57 | 58 | ## Kerberos Secure Cluster Option 59 | 60 | This automation supports using a local KDC deployed on the Cloudera Manager instance for secure cluster operation. Please read the scripts [README](scripts/README.md) for information regarding how to set these parameters prior to deployment if desired. This can be toggled during ORM stack setup using the schema. 61 | 62 | Also - for cluster management using Kerberos, you will need to manually create at a minimum the HDFS Superuser Principal as [detailed here](https://www.cloudera.com/documentation/enterprise/latest/topics/cm_sg_using_cm_sec_config.html#create-hdfs-superuser) after deployment. 63 | 64 | ## High Availability 65 | 66 | High Availability for HDFS services is also offered as part of the deployment process. This can be toggled during ORM stack setup using the Schema. 67 | 68 | ## Metadata and MySQL 69 | 70 | You can customize the default root password for MySQL by editing the source script [cms_mysql.sh](scripts/cms_mysql.sh#L188). For the various Cloudera databases, random passwords are generated and used. These are stored in a flat file on the Utility host for use at deployment time. This file should be removed after you notate/change the pre-generated passwords, it is located here on the Utility node: `/etc/mysql/mysql.pw` 71 | 72 | ## Object Storage Integration 73 | 74 | Object Storage can also be leveraged by setting S3 compatability paramaters in the Python deployment script. Details can be found in the [header section](https://github.com/oracle-quickstart/oci-cloudera/blob/8af97b91fb50cd77262c97580454137c2955dd4e/scripts/deploy_on_oci.py#L79-L86). You will need to setup the appropriate S3 compatability pre-requisites as [detailed here](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) for this to work. 75 | 76 | ## Architecture Diagram 77 | Here is a diagram showing what is typically deployed using this template. Note that resources are automatically distributed among Fault Domains in an Availability Domain to ensure fault tolerance. Additional workers deployed will stripe between the 3 fault domains in sequence starting with the Fault Domain 1 and incrementing sequentially. 78 | 79 | ![Deployment Architecture Diagram](images/deployment_architecture.png) 80 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd $(dirname $0) 3 | SCRIPT_DIR=$(pwd) 4 | 5 | echo "-->Cleaning build folder" 6 | rm -rf $SCRIPT_DIR/build 7 | mkdir -p $SCRIPT_DIR/build 8 | 9 | echo "-->Creating BYOL stack" 10 | folder=$(mktemp -d "cloudera-XXXXX") 11 | mkdir -p $folder 12 | cd $folder 13 | echo "-->Copying Terraform stack" 14 | rsync -apxrv $SCRIPT_DIR/ --exclude=build --exclude=.github --exclude=.git --exclude=$folder . 15 | echo "-->Cleanup .terraform" 16 | rm -rf .terraform 17 | echo "-->Showing contents of build root:" 18 | ls -la 19 | echo "-->Creating ZIP archive" 20 | zip -r $SCRIPT_DIR/build/cloudera-byol.zip * 21 | echo "-->Cleanup $SCRIPT_DIR" 22 | cd $SCRIPT_DIR 23 | rm -rf $folder 24 | echo "-->Showing contents of $SCRIPT_DIR/build" 25 | ls -la $SCRIPT_DIR/build 26 | -------------------------------------------------------------------------------- /compute.tf: -------------------------------------------------------------------------------- 1 | data "oci_core_vcn" "vcn_info" { 2 | vcn_id = "${var.useExistingVcn ? var.myVcn : module.network.vcn-id}" 3 | } 4 | 5 | data "oci_core_subnet" "cluster_subnet" { 6 | subnet_id = "${var.useExistingVcn ? var.clusterSubnet : module.network.private-id}" 7 | } 8 | 9 | data "oci_core_subnet" "bastion_subnet" { 10 | subnet_id = "${var.useExistingVcn ? var.bastionSubnet : module.network.bastion-id}" 11 | } 12 | 13 | data "oci_core_subnet" "utility_subnet" { 14 | subnet_id = "${var.useExistingVcn ? var.utilitySubnet : module.network.public-id}" 15 | } 16 | 17 | data "null_data_source" "values" { 18 | inputs = { 19 | cm_default = "cloudera-utility-1.${data.oci_core_subnet.utility_subnet.dns_label}.${data.oci_core_vcn.vcn_info.vcn_domain_name}" 20 | worker_domain = "${data.oci_core_subnet.cluster_subnet.dns_label}.${data.oci_core_vcn.vcn_info.vcn_domain_name}" 21 | } 22 | } 23 | 24 | data "null_data_source" "vpus" { 25 | inputs = { 26 | block_vpus = "${var.block_volume_high_performance ? 20 : 0}" 27 | } 28 | } 29 | 30 | module "bastion" { 31 | source = "./modules/bastion" 32 | instances = "${var.use_edge_nodes ? var.bastion_node_count : 0}" 33 | region = "${var.region}" 34 | compartment_ocid = "${var.compartment_ocid}" 35 | subnet_id = "${var.useExistingVcn ? var.bastionSubnet : module.network.bastion-id}" 36 | availability_domain = "${var.availability_domain}" 37 | image_ocid = "${var.cloudera_version == "7.0.3.0" ? var.CentOSImageOCID[var.region] : var.OELImageOCID[var.region]}" 38 | ssh_public_key = "${var.provide_ssh_key ? var.ssh_provided_key : tls_private_key.key.public_key_openssh}" 39 | bastion_instance_shape = "${var.bastion_instance_shape}" 40 | log_volume_size_in_gbs = "${var.log_volume_size_in_gbs}" 41 | cloudera_volume_size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 42 | user_data = "${base64encode(file("scripts/boot.sh"))}" 43 | cloudera_manager = "${data.null_data_source.values.outputs["cm_default"]}" 44 | cm_version = "${var.cm_version}" 45 | cloudera_version = "${var.cloudera_version}" 46 | } 47 | 48 | module "utility" { 49 | source = "./modules/utility" 50 | instances = "1" 51 | region = "${var.region}" 52 | compartment_ocid = "${var.compartment_ocid}" 53 | subnet_id = "${var.useExistingVcn ? var.utilitySubnet : module.network.public-id}" 54 | availability_domain = "${var.availability_domain}" 55 | image_ocid = "${var.cloudera_version == "7.0.3.0" ? var.CentOSImageOCID[var.region] : var.OELImageOCID[var.region]}" 56 | ssh_public_key = "${var.provide_ssh_key ? var.ssh_provided_key : tls_private_key.key.public_key_openssh}" 57 | utility_instance_shape = "${var.utility_instance_shape}" 58 | log_volume_size_in_gbs = "${var.log_volume_size_in_gbs}" 59 | cloudera_volume_size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 60 | user_data = "${base64gzip(file("scripts/cloudera_manager_boot.sh"))}" 61 | cm_install = "${var.meta_db_type == "mysql" ? base64gzip(file("scripts/cms_mysql.sh")) : base64gzip(file("scripts/cms_postgres.sh"))}" 62 | deploy_on_oci = "${base64gzip(file("scripts/deploy_on_oci.py"))}" 63 | cloudera_manager = "${data.null_data_source.values.outputs["cm_default"]}" 64 | cm_version = "${var.cm_version}" 65 | cloudera_version = "${var.cloudera_version}" 66 | worker_shape = "${var.worker_instance_shape}" 67 | block_volume_count = "${var.enable_block_volumes ? var.block_volumes_per_worker : 0}" 68 | hdfs_ha = "${var.hdfs_ha}" 69 | secure_cluster = "${var.secure_cluster}" 70 | cluster_name = "${var.cluster_name}" 71 | cluster_subnet = "${data.oci_core_subnet.cluster_subnet.dns_label}" 72 | bastion_subnet = "${data.oci_core_subnet.bastion_subnet.dns_label}" 73 | utility_subnet = "${data.oci_core_subnet.utility_subnet.dns_label}" 74 | meta_db_type = "${var.meta_db_type}" 75 | cm_username = "${var.cm_username}" 76 | cm_password = "${var.cm_password}" 77 | vcore_ratio = "${var.vcore_ratio}" 78 | svc_ATLAS = "${var.svc_ATLAS}" 79 | svc_HBASE = "${var.svc_HBASE}" 80 | svc_HDFS = "${var.svc_HDFS}" 81 | svc_HIVE = "${var.svc_HIVE}" 82 | svc_IMPALA = "${var.svc_IMPALA}" 83 | svc_KAFKA = "${var.svc_KAFKA}" 84 | svc_OOZIE = "${var.svc_OOZIE}" 85 | svc_RANGER = "${var.svc_RANGER}" 86 | svc_SOLR = "${var.svc_SOLR}" 87 | svc_SPARK_ON_YARN = "${var.svc_SPARK_ON_YARN}" 88 | svc_SQOOP_CLIENT = "${var.svc_SQOOP_CLIENT}" 89 | svc_YARN = "${var.svc_YARN}" 90 | enable_debug = "${var.enable_debug}" 91 | rangeradmin_password = "${var.rangeradmin_password}" 92 | yarn_scheduler = "${var.yarn_scheduler}" 93 | } 94 | 95 | module "master" { 96 | source = "./modules/master" 97 | instances = "${var.master_node_count}" 98 | region = "${var.region}" 99 | compartment_ocid = "${var.compartment_ocid}" 100 | subnet_id = "${var.useExistingVcn ? var.clusterSubnet : module.network.private-id}" 101 | availability_domain = "${var.availability_domain}" 102 | image_ocid = "${var.cloudera_version == "7.0.3.0" ? var.CentOSImageOCID[var.region] : var.OELImageOCID[var.region]}" 103 | ssh_public_key = "${var.provide_ssh_key ? var.ssh_provided_key : tls_private_key.key.public_key_openssh}" 104 | master_instance_shape = "${var.master_instance_shape}" 105 | log_volume_size_in_gbs = "${var.log_volume_size_in_gbs}" 106 | cloudera_volume_size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 107 | user_data = "${base64encode(file("scripts/boot.sh"))}" 108 | cloudera_manager = "${data.null_data_source.values.outputs["cm_default"]}" 109 | cm_version = "${var.cm_version}" 110 | cloudera_version = "${var.cloudera_version}" 111 | } 112 | 113 | module "worker" { 114 | source = "./modules/worker" 115 | instances = "${var.worker_node_count}" 116 | region = "${var.region}" 117 | compartment_ocid = "${var.compartment_ocid}" 118 | subnet_id = "${var.useExistingVcn ? var.clusterSubnet : module.network.private-id}" 119 | blockvolume_subnet_id = "${var.useExistingVcn ? var.blockvolumeSubnet : module.network.blockvolume-id}" 120 | availability_domain = "${var.availability_domain}" 121 | image_ocid = "${var.cloudera_version == "7.0.3.0" ? var.CentOSImageOCID[var.region] : var.OELImageOCID[var.region]}" 122 | ssh_public_key = "${var.provide_ssh_key ? var.ssh_provided_key : tls_private_key.key.public_key_openssh}" 123 | worker_instance_shape = "${var.worker_instance_shape}" 124 | log_volume_size_in_gbs = "${var.log_volume_size_in_gbs}" 125 | cloudera_volume_size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 126 | block_volumes_per_worker = "${var.enable_block_volumes ? var.block_volumes_per_worker : 0}" 127 | data_blocksize_in_gbs = "${var.data_blocksize_in_gbs}" 128 | user_data = "${base64encode(file("scripts/boot.sh"))}" 129 | cloudera_manager = "${data.null_data_source.values.outputs["cm_default"]}" 130 | cm_version = "${var.cm_version}" 131 | cloudera_version = "${var.cloudera_version}" 132 | block_volume_count = "${var.enable_block_volumes ? var.block_volumes_per_worker : 0}" 133 | vpus_per_gb = "${var.customize_block_volume_performance ? data.null_data_source.vpus.outputs["block_vpus"] : 10}" 134 | objectstoreRAID = "${var.objectstoreRAID}" 135 | enable_secondary_vnic = "${var.enable_secondary_vnic}" 136 | secondary_vnic_count = "${var.enable_secondary_vnic ? 1 : 0}" 137 | worker_domain = "${data.null_data_source.values.outputs["worker_domain"]}" 138 | } 139 | -------------------------------------------------------------------------------- /data.tf: -------------------------------------------------------------------------------- 1 | data "oci_identity_availability_domains" "ADs" { 2 | compartment_id = "${var.tenancy_ocid}" 3 | } 4 | -------------------------------------------------------------------------------- /iam.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "key" { 2 | algorithm = "RSA" 3 | } 4 | 5 | resource "local_file" "key_file" { 6 | filename = "${path.module}/key.pem" 7 | content = "${tls_private_key.key.private_key_pem}" 8 | } 9 | -------------------------------------------------------------------------------- /images/01 - manager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-cloudera/357d5f15f0336df38779a6fe14ae12c4a88546bd/images/01 - manager.png -------------------------------------------------------------------------------- /images/02 - home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-cloudera/357d5f15f0336df38779a6fe14ae12c4a88546bd/images/02 - home.png -------------------------------------------------------------------------------- /images/RM_variables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-cloudera/357d5f15f0336df38779a6fe14ae12c4a88546bd/images/RM_variables.png -------------------------------------------------------------------------------- /images/cloudera_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-cloudera/357d5f15f0336df38779a6fe14ae12c4a88546bd/images/cloudera_logo.png -------------------------------------------------------------------------------- /images/deployment_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-cloudera/357d5f15f0336df38779a6fe14ae12c4a88546bd/images/deployment_architecture.png -------------------------------------------------------------------------------- /modules/bastion/main.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_instance" "Bastion" { 2 | count = "${var.instances}" 3 | availability_domain = "${var.availability_domain}" 4 | compartment_id = "${var.compartment_ocid}" 5 | shape = "${var.bastion_instance_shape}" 6 | display_name = "Cloudera Bastion ${format("%01d", count.index+1)}" 7 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 8 | 9 | source_details { 10 | source_type = "image" 11 | source_id = "${var.image_ocid}" 12 | } 13 | 14 | create_vnic_details { 15 | subnet_id = "${var.subnet_id}" 16 | display_name = "Cloudera Bastion ${format("%01d", count.index+1)}" 17 | hostname_label = "Cloudera-Bastion-${format("%01d", count.index+1)}" 18 | assign_public_ip = "${var.hide_private_subnet ? true : false}" 19 | } 20 | 21 | metadata = { 22 | ssh_authorized_keys = "${var.ssh_public_key}" 23 | user_data = "${var.user_data}" 24 | cloudera_manager = "${var.cloudera_manager}" 25 | cloudera_version = "${var.cloudera_version}" 26 | cm_version = "${var.cm_version}" 27 | } 28 | 29 | timeouts { 30 | create = "30m" 31 | } 32 | } 33 | // Block Volume Creation for Bastion 34 | 35 | # Log Volume for /var/log/cloudera 36 | resource "oci_core_volume" "BastionLogVolume" { 37 | count = "${var.instances}" 38 | availability_domain = "${var.availability_domain}" 39 | compartment_id = "${var.compartment_ocid}" 40 | display_name = "Bastion ${format("%01d", count.index+1)} Log Data" 41 | size_in_gbs = "${var.log_volume_size_in_gbs}" 42 | } 43 | 44 | resource "oci_core_volume_attachment" "BastionLogAttachment" { 45 | count = "${var.instances}" 46 | attachment_type = "iscsi" 47 | instance_id = "${oci_core_instance.Bastion.*.id[count.index]}" 48 | volume_id = "${oci_core_volume.BastionLogVolume.*.id[count.index]}" 49 | device = "/dev/oracleoci/oraclevdb" 50 | } 51 | 52 | # Data Volume for /opt/cloudera 53 | resource "oci_core_volume" "BastionClouderaVolume" { 54 | count = "${var.instances}" 55 | availability_domain = "${var.availability_domain}" 56 | compartment_id = "${var.compartment_ocid}" 57 | display_name = "Bastion ${format("%01d", count.index+1)} Cloudera Data" 58 | size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 59 | } 60 | 61 | resource "oci_core_volume_attachment" "BastionClouderaAttachment" { 62 | count = "${var.instances}" 63 | attachment_type = "iscsi" 64 | instance_id = "${oci_core_instance.Bastion.*.id[count.index]}" 65 | volume_id = "${oci_core_volume.BastionClouderaVolume.*.id[count.index]}" 66 | device = "/dev/oracleoci/oraclevdc" 67 | } 68 | 69 | -------------------------------------------------------------------------------- /modules/bastion/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # Environmental variables 3 | # You probably want to define these as environmental variables. 4 | # Instructions on that are here: https://github.com/oci-quickstart/oci-prerequisites 5 | # --------------------------------------------------------------------------------------------------------------------- 6 | 7 | variable "region" {} 8 | variable "compartment_ocid" {} 9 | variable "ssh_public_key" {} 10 | variable "instances" {} 11 | variable "subnet_id" {} 12 | variable "user_data" {} 13 | variable "image_ocid" {} 14 | variable "cm_version" {} 15 | variable "cloudera_version" {} 16 | variable "cloudera_manager" {} 17 | variable "hide_private_subnet" { 18 | default = "true" 19 | } 20 | 21 | # --------------------------------------------------------------------------------------------------------------------- 22 | # Optional variables 23 | # You can modify these. 24 | # --------------------------------------------------------------------------------------------------------------------- 25 | 26 | variable "availability_domain" { 27 | default = "2" 28 | } 29 | 30 | # Size for Cloudera Log Volumes across all hosts deployed to /var/log/cloudera 31 | 32 | variable "log_volume_size_in_gbs" { 33 | default = "200" 34 | } 35 | 36 | # Size for Volume across all hosts deployed to /opt/cloudera 37 | 38 | variable "cloudera_volume_size_in_gbs" { 39 | default = "300" 40 | } 41 | 42 | # 43 | # Set Cluster Shapes in this section 44 | # 45 | 46 | variable "bastion_instance_shape" { 47 | default = "VM.Standard2.8" 48 | } 49 | 50 | # --------------------------------------------------------------------------------------------------------------------- 51 | # Constants 52 | # You probably don't need to change these. 53 | # --------------------------------------------------------------------------------------------------------------------- 54 | 55 | -------------------------------------------------------------------------------- /modules/master/main.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_instance" "Master" { 2 | count = "${var.instances}" 3 | availability_domain = "${var.availability_domain}" 4 | compartment_id = "${var.compartment_ocid}" 5 | shape = "${var.master_instance_shape}" 6 | display_name = "Cloudera Master ${format("%01d", count.index+1)}" 7 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 8 | 9 | source_details { 10 | source_type = "image" 11 | source_id = "${var.image_ocid}" 12 | } 13 | 14 | create_vnic_details { 15 | subnet_id = "${var.subnet_id}" 16 | display_name = "Cloudera Master ${format("%01d", count.index+1)}" 17 | hostname_label = "Cloudera-Master-${format("%01d", count.index+1)}" 18 | assign_public_ip = "${var.hide_public_subnet ? false : true}" 19 | } 20 | 21 | metadata = { 22 | ssh_authorized_keys = "${var.ssh_public_key}" 23 | user_data = "${var.user_data}" 24 | cloudera_manager = "${var.cloudera_manager}" 25 | cloudera_version = "${var.cloudera_version}" 26 | cm_version = "${var.cm_version}" 27 | } 28 | 29 | timeouts { 30 | create = "30m" 31 | } 32 | } 33 | 34 | // Block Volume Creation for Master 35 | 36 | # Log Volume for /var/log/cloudera 37 | resource "oci_core_volume" "MasterLogVolume" { 38 | count = "${var.instances}" 39 | availability_domain = "${var.availability_domain}" 40 | compartment_id = "${var.compartment_ocid}" 41 | display_name = "Cloudera Master ${format("%01d", count.index+1)} Log Data" 42 | size_in_gbs = "${var.log_volume_size_in_gbs}" 43 | } 44 | 45 | resource "oci_core_volume_attachment" "MasterLogAttachment" { 46 | count = "${var.instances}" 47 | attachment_type = "iscsi" 48 | instance_id = "${oci_core_instance.Master[count.index].id}" 49 | volume_id = "${oci_core_volume.MasterLogVolume[count.index].id}" 50 | device = "/dev/oracleoci/oraclevdb" 51 | } 52 | 53 | # Data Volume for /opt/cloudera 54 | resource "oci_core_volume" "MasterClouderaVolume" { 55 | count = "${var.instances}" 56 | availability_domain = "${var.availability_domain}" 57 | compartment_id = "${var.compartment_ocid}" 58 | display_name = "Cloudera Master ${format("%01d", count.index+1)} Cloudera Data" 59 | size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 60 | } 61 | 62 | resource "oci_core_volume_attachment" "MasterClouderaAttachment" { 63 | count = "${var.instances}" 64 | attachment_type = "iscsi" 65 | instance_id = "${oci_core_instance.Master[count.index].id}" 66 | volume_id = "${oci_core_volume.MasterClouderaVolume[count.index].id}" 67 | device = "/dev/oracleoci/oraclevdc" 68 | } 69 | 70 | # Data Volume for /data (Name & SecondaryName) 71 | resource "oci_core_volume" "MasterNNVolume" { 72 | count = "${var.instances}" 73 | availability_domain = "${var.availability_domain}" 74 | compartment_id = "${var.compartment_ocid}" 75 | display_name = "Cloudera Master ${format("%01d", count.index+1)} Journal Data" 76 | size_in_gbs = "${var.nn_volume_size_in_gbs}" 77 | } 78 | 79 | resource "oci_core_volume_attachment" "MasterNNAttachment" { 80 | count = "${var.instances}" 81 | attachment_type = "iscsi" 82 | instance_id = "${oci_core_instance.Master[count.index].id}" 83 | volume_id = "${oci_core_volume.MasterNNVolume[count.index].id}" 84 | device = "/dev/oracleoci/oraclevdd" 85 | } 86 | 87 | -------------------------------------------------------------------------------- /modules/master/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # Environmental variables 3 | # You probably want to define these as environmental variables. 4 | # Instructions on that are here: https://github.com/oci-quickstart/oci-prerequisites 5 | # --------------------------------------------------------------------------------------------------------------------- 6 | 7 | variable "region" {} 8 | variable "compartment_ocid" {} 9 | variable "ssh_public_key" {} 10 | variable "instances" {} 11 | variable "subnet_id" {} 12 | variable "user_data" {} 13 | variable "image_ocid" {} 14 | variable "cm_version" {} 15 | variable "cloudera_version" {} 16 | variable "cloudera_manager" {} 17 | variable "hide_public_subnet" { 18 | default = "true" 19 | } 20 | 21 | # --------------------------------------------------------------------------------------------------------------------- 22 | # Optional variables 23 | # You can modify these. 24 | # --------------------------------------------------------------------------------------------------------------------- 25 | 26 | variable "availability_domain" { 27 | default = "2" 28 | } 29 | 30 | # Number of Master Nodes in the Cluster 31 | # For Scaling See https://www.cloudera.com/documentation/enterprise/latest/topics/cm_ig_host_allocations.html 32 | 33 | variable "master_node_count" { 34 | default = "2" 35 | } 36 | 37 | # Size for Cloudera Log Volumes across all hosts deployed to /var/log/cloudera 38 | 39 | variable "log_volume_size_in_gbs" { 40 | default = "200" 41 | } 42 | 43 | # Size for Volume across all hosts deployed to /opt/cloudera 44 | 45 | variable "cloudera_volume_size_in_gbs" { 46 | default = "300" 47 | } 48 | 49 | # Size for NameNode and SecondaryNameNode data volume (Journal Data) 50 | 51 | variable "nn_volume_size_in_gbs" { 52 | default = "500" 53 | } 54 | 55 | # 56 | # Set Cluster Shapes in this section 57 | # 58 | 59 | variable "master_instance_shape" { 60 | default = "VM.Standard2.8" 61 | } 62 | 63 | # --------------------------------------------------------------------------------------------------------------------- 64 | # Constants 65 | # You probably don't need to change these. 66 | # --------------------------------------------------------------------------------------------------------------------- 67 | -------------------------------------------------------------------------------- /modules/network/data.tf: -------------------------------------------------------------------------------- 1 | data "oci_identity_availability_domains" "ADs" { 2 | count = var.useExistingVcn ? 0 : 1 3 | compartment_id = "${var.tenancy_ocid}" 4 | } 5 | 6 | data "oci_core_services" "all_svcs_moniker" { 7 | count = var.useExistingVcn ? 0 : 1 8 | filter { 9 | name = "name" 10 | values = ["All .* Services In Oracle Services Network"] 11 | regex = true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /modules/network/main.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_vcn" "cloudera_vcn" { 2 | count = var.useExistingVcn ? 0 : 1 3 | cidr_block = "${var.VPC_CIDR}" 4 | compartment_id = "${var.compartment_ocid}" 5 | display_name = "cloudera_vcn" 6 | dns_label = "${var.vcn_dns_label}" 7 | } 8 | 9 | resource "oci_core_internet_gateway" "cloudera_internet_gateway" { 10 | count = var.useExistingVcn ? 0 : 1 11 | compartment_id = "${var.compartment_ocid}" 12 | display_name = "cloudera_internet_gateway" 13 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 14 | } 15 | 16 | resource "oci_core_nat_gateway" "nat_gateway" { 17 | count = var.useExistingVcn ? 0 : 1 18 | compartment_id = "${var.compartment_ocid}" 19 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 20 | display_name = "nat_gateway" 21 | } 22 | 23 | resource "oci_core_service_gateway" "cloudera_service_gateway" { 24 | count = var.useExistingVcn ? 0 : 1 25 | compartment_id = "${var.compartment_ocid}" 26 | services { 27 | service_id = "${lookup(data.oci_core_services.all_svcs_moniker[count.index].services[0], "id")}" 28 | } 29 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 30 | display_name = "Cloudera Service Gateway" 31 | } 32 | 33 | resource "oci_core_route_table" "RouteForComplete" { 34 | count = var.useExistingVcn ? 0 : 1 35 | compartment_id = "${var.compartment_ocid}" 36 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 37 | display_name = "RouteTableForComplete" 38 | 39 | route_rules { 40 | destination = "0.0.0.0/0" 41 | destination_type = "CIDR_BLOCK" 42 | network_entity_id = "${oci_core_internet_gateway.cloudera_internet_gateway.*.id[count.index]}" 43 | } 44 | } 45 | 46 | resource "oci_core_route_table" "private" { 47 | count = var.useExistingVcn ? 0 : 1 48 | compartment_id = "${var.compartment_ocid}" 49 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 50 | display_name = "private" 51 | 52 | route_rules { 53 | destination = "${var.oci_service_gateway}" 54 | destination_type = "SERVICE_CIDR_BLOCK" 55 | network_entity_id = "${oci_core_service_gateway.cloudera_service_gateway.*.id[count.index]}" 56 | } 57 | 58 | route_rules { 59 | destination = "0.0.0.0/0" 60 | destination_type = "CIDR_BLOCK" 61 | network_entity_id = "${oci_core_nat_gateway.nat_gateway.*.id[count.index]}" 62 | } 63 | } 64 | 65 | resource "oci_core_route_table" "blockvolume" { 66 | count = var.useExistingVcn ? 0 : 1 67 | compartment_id = "${var.compartment_ocid}" 68 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 69 | display_name = "blockvolume" 70 | 71 | route_rules { 72 | destination = "${var.oci_service_gateway}" 73 | destination_type = "SERVICE_CIDR_BLOCK" 74 | network_entity_id = "${oci_core_service_gateway.cloudera_service_gateway.*.id[count.index]}" 75 | } 76 | 77 | route_rules { 78 | destination = "0.0.0.0/0" 79 | destination_type = "CIDR_BLOCK" 80 | network_entity_id = "${oci_core_nat_gateway.nat_gateway.*.id[count.index]}" 81 | } 82 | } 83 | 84 | resource "oci_core_security_list" "PublicSubnet" { 85 | count = var.useExistingVcn ? 0 : 1 86 | compartment_id = "${var.compartment_ocid}" 87 | display_name = "Public Subnet" 88 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 89 | 90 | egress_security_rules { 91 | destination = "0.0.0.0/0" 92 | protocol = "6" 93 | } 94 | 95 | ingress_security_rules { 96 | tcp_options { 97 | max = 7180 98 | min = 7180 99 | } 100 | 101 | protocol = "6" 102 | source = "0.0.0.0/0" 103 | } 104 | ingress_security_rules { 105 | tcp_options { 106 | max = 18088 107 | min = 18088 108 | } 109 | 110 | protocol = "6" 111 | source = "0.0.0.0/0" 112 | } 113 | ingress_security_rules { 114 | tcp_options { 115 | max = 19888 116 | min = 19888 117 | } 118 | 119 | protocol = "6" 120 | source = "0.0.0.0/0" 121 | } 122 | ingress_security_rules { 123 | tcp_options { 124 | max = 22 125 | min = 22 126 | } 127 | 128 | protocol = "6" 129 | source = "0.0.0.0/0" 130 | } 131 | ingress_security_rules { 132 | protocol = "6" 133 | source = "${var.VPC_CIDR}" 134 | } 135 | } 136 | 137 | resource "oci_core_security_list" "PrivateSubnet" { 138 | count = var.useExistingVcn ? 0 : 1 139 | compartment_id = "${var.compartment_ocid}" 140 | display_name = "Private" 141 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 142 | 143 | egress_security_rules { 144 | destination = "0.0.0.0/0" 145 | protocol = "6" 146 | } 147 | egress_security_rules { 148 | protocol = "6" 149 | destination = "${var.VPC_CIDR}" 150 | } 151 | 152 | ingress_security_rules { 153 | protocol = "6" 154 | source = "${var.VPC_CIDR}" 155 | } 156 | } 157 | 158 | resource "oci_core_security_list" "BVSubnet" { 159 | count = var.useExistingVcn ? 0 : 1 160 | compartment_id = "${var.compartment_ocid}" 161 | display_name = "BlockVolume" 162 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 163 | 164 | egress_security_rules { 165 | destination = "0.0.0.0/0" 166 | protocol = "6" 167 | } 168 | egress_security_rules { 169 | protocol = "6" 170 | destination = "${var.VPC_CIDR}" 171 | } 172 | 173 | ingress_security_rules { 174 | protocol = "6" 175 | source = "${var.VPC_CIDR}" 176 | } 177 | } 178 | 179 | resource "oci_core_security_list" "BastionSubnet" { 180 | count = var.useExistingVcn ? 0 : 1 181 | compartment_id = "${var.compartment_ocid}" 182 | display_name = "Bastion" 183 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 184 | 185 | egress_security_rules { 186 | protocol = "6" 187 | destination = "0.0.0.0/0" 188 | } 189 | 190 | ingress_security_rules { 191 | tcp_options { 192 | max = 22 193 | min = 22 194 | } 195 | 196 | protocol = "6" 197 | source = "0.0.0.0/0" 198 | } 199 | ingress_security_rules { 200 | protocol = "6" 201 | source = "${var.VPC_CIDR}" 202 | } 203 | } 204 | 205 | resource "oci_core_subnet" "public" { 206 | count = var.useExistingVcn ? 0 : 1 207 | availability_domain = "${var.availability_domain}" 208 | cidr_block = "${var.custom_cidrs ? var.public_cidr : cidrsubnet(var.VPC_CIDR, 8, 1)}" 209 | display_name = "public" 210 | compartment_id = "${var.compartment_ocid}" 211 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 212 | route_table_id = "${oci_core_route_table.RouteForComplete[count.index].id}" 213 | security_list_ids = ["${oci_core_security_list.PublicSubnet.*.id[count.index]}"] 214 | dhcp_options_id = "${oci_core_vcn.cloudera_vcn[count.index].default_dhcp_options_id}" 215 | dns_label = "public" 216 | } 217 | 218 | resource "oci_core_subnet" "private" { 219 | count = var.useExistingVcn ? 0 : 1 220 | availability_domain = "${var.availability_domain}" 221 | cidr_block = "${var.custom_cidrs ? var.private_cidr : cidrsubnet(var.VPC_CIDR, 8, 2)}" 222 | display_name = "private" 223 | compartment_id = "${var.compartment_ocid}" 224 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 225 | route_table_id = "${oci_core_route_table.private[count.index].id}" 226 | security_list_ids = ["${oci_core_security_list.PrivateSubnet.*.id[count.index]}"] 227 | dhcp_options_id = "${oci_core_vcn.cloudera_vcn[count.index].default_dhcp_options_id}" 228 | prohibit_public_ip_on_vnic = "true" 229 | dns_label = "private" 230 | } 231 | 232 | resource "oci_core_subnet" "bastion" { 233 | count = var.useExistingVcn ? 0 : 1 234 | availability_domain = "${var.availability_domain}" 235 | cidr_block = "${var.custom_cidrs ? var.edge_cidr : cidrsubnet(var.VPC_CIDR, 8, 3)}" 236 | display_name = "bastion" 237 | compartment_id = "${var.compartment_ocid}" 238 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 239 | route_table_id = "${oci_core_route_table.RouteForComplete[count.index].id}" 240 | security_list_ids = ["${oci_core_security_list.BastionSubnet.*.id[count.index]}"] 241 | dhcp_options_id = "${oci_core_vcn.cloudera_vcn[count.index].default_dhcp_options_id}" 242 | dns_label = "bastion" 243 | } 244 | 245 | resource "oci_core_subnet" "blockvolume" { 246 | count = var.useExistingVcn ? 0 : 1 247 | availability_domain = "${var.availability_domain}" 248 | cidr_block = "${var.custom_cidrs ? var.blockvolume_cidr : cidrsubnet(var.VPC_CIDR, 8, 4)}" 249 | display_name = "blockvolume" 250 | compartment_id = "${var.compartment_ocid}" 251 | vcn_id = "${var.useExistingVcn ? var.custom_vcn[0] : oci_core_vcn.cloudera_vcn.0.id}" 252 | route_table_id = "${oci_core_route_table.private[count.index].id}" 253 | security_list_ids = ["${oci_core_security_list.BVSubnet.*.id[count.index]}"] 254 | dhcp_options_id = "${oci_core_vcn.cloudera_vcn[count.index].default_dhcp_options_id}" 255 | dns_label = "blockvolume" 256 | } 257 | -------------------------------------------------------------------------------- /modules/network/outputs.tf: -------------------------------------------------------------------------------- 1 | output "vcn-id" { 2 | value = "${var.useExistingVcn ? var.myVcn : oci_core_vcn.cloudera_vcn.0.id}" 3 | } 4 | 5 | output "private-id" { 6 | value = "${var.useExistingVcn ? var.clusterSubnet : oci_core_subnet.private.0.id}" 7 | } 8 | 9 | output "public-id" { 10 | value = "${var.useExistingVcn ? var.utilitySubnet : oci_core_subnet.public.0.id}" 11 | } 12 | 13 | output "bastion-id" { 14 | value = "${var.useExistingVcn ? var.bastionSubnet : oci_core_subnet.bastion.0.id}" 15 | } 16 | 17 | output "blockvolume-id" { 18 | value = "${var.useExistingVcn ? var.blockvolumeSubnet : oci_core_subnet.blockvolume.0.id}" 19 | } 20 | -------------------------------------------------------------------------------- /modules/network/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # Environmental variables 3 | # You probably want to define these as environmental variables. 4 | # Instructions on that are here: https://github.com/oci-quickstart/oci-prerequisites 5 | # --------------------------------------------------------------------------------------------------------------------- 6 | 7 | variable "tenancy_ocid" {} 8 | variable "compartment_ocid" {} 9 | variable "region" {} 10 | variable "oci_service_gateway" {} 11 | variable "VPC_CIDR" {} 12 | variable "useExistingVcn" {} 13 | variable "custom_vcn" { 14 | type = list(string) 15 | default = [" "] 16 | } 17 | variable "custom_cidrs" { 18 | default = "false" 19 | } 20 | variable "vcn_dns_label" { 21 | default = "clouderavcn" 22 | } 23 | variable "edge_cidr" {} 24 | variable "public_cidr" {} 25 | variable "private_cidr" {} 26 | variable "blockvolume_cidr" {} 27 | variable "enable_secondary_vnic" {} 28 | variable "myVcn" {} 29 | variable "clusterSubnet" { 30 | default = " " 31 | } 32 | variable "bastionSubnet" { 33 | default = " " 34 | } 35 | variable "utilitySubnet" { 36 | default = " " 37 | } 38 | variable "blockvolumeSubnet" { 39 | default = " " 40 | } 41 | # --------------------------------------------------------------------------------------------------------------------- 42 | # Optional variables 43 | # You can modify these. 44 | # --------------------------------------------------------------------------------------------------------------------- 45 | 46 | variable "availability_domain" { 47 | default = "2" 48 | } 49 | 50 | 51 | -------------------------------------------------------------------------------- /modules/utility/data.tf: -------------------------------------------------------------------------------- 1 | 2 | data "oci_core_vnic_attachments" "utility_node_vnics" { 3 | compartment_id = "${var.compartment_ocid}" 4 | availability_domain = "${var.availability_domain}" 5 | instance_id = "${oci_core_instance.Utility.id}" 6 | } 7 | 8 | data "oci_core_vnic" "utility_node_vnic" { 9 | vnic_id = "${lookup(data.oci_core_vnic_attachments.utility_node_vnics.vnic_attachments[0],"vnic_id")}" 10 | } 11 | -------------------------------------------------------------------------------- /modules/utility/main.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_instance" "Utility" { 2 | availability_domain = "${var.availability_domain}" 3 | compartment_id = "${var.compartment_ocid}" 4 | shape = "${var.utility_instance_shape}" 5 | display_name = "Cloudera Utility-1" 6 | fault_domain = "FAULT-DOMAIN-3" 7 | 8 | source_details { 9 | source_type = "image" 10 | source_id = "${var.image_ocid}" 11 | } 12 | 13 | create_vnic_details { 14 | subnet_id = "${var.subnet_id}" 15 | display_name = "Cloudera Utility-1" 16 | hostname_label = "Cloudera-Utility-1" 17 | assign_public_ip = "${var.hide_private_subnet ? true : false}" 18 | } 19 | 20 | metadata = { 21 | ssh_authorized_keys = "${var.ssh_public_key}" 22 | user_data = "${var.user_data}" 23 | } 24 | 25 | extended_metadata = { 26 | cm_install = "${var.cm_install}" 27 | meta_db_type = "${var.meta_db_type}" 28 | deploy_on_oci = "${var.deploy_on_oci}" 29 | cloudera_manager = "${var.cloudera_manager}" 30 | cloudera_version = "${var.cloudera_version}" 31 | cm_version = "${var.cm_version}" 32 | worker_shape = "${var.worker_shape}" 33 | block_volume_count = "${var.block_volume_count}" 34 | secure_cluster = "${var.secure_cluster}" 35 | hdfs_ha = "${var.hdfs_ha}" 36 | cluster_name = "${var.cluster_name}" 37 | cluster_subnet = "${var.cluster_subnet}" 38 | bastion_subnet = "${var.bastion_subnet}" 39 | utility_subnet = "${var.utility_subnet}" 40 | cm_username = "${var.cm_username}" 41 | cm_password = "${var.cm_password}" 42 | vcore_ratio = "${var.vcore_ratio}" 43 | svc_ATLAS = "${var.svc_ATLAS}" 44 | svc_HBASE = "${var.svc_HBASE}" 45 | svc_HDFS = "${var.svc_HDFS}" 46 | svc_HIVE = "${var.svc_HIVE}" 47 | svc_IMPALA = "${var.svc_IMPALA}" 48 | svc_KAFKA = "${var.svc_KAFKA}" 49 | svc_OOZIE = "${var.svc_OOZIE}" 50 | svc_RANGER = "${var.svc_RANGER}" 51 | svc_SOLR = "${var.svc_SOLR}" 52 | svc_SPARK_ON_YARN = "${var.svc_SPARK_ON_YARN}" 53 | svc_SQOOP_CLIENT = "${var.svc_SQOOP_CLIENT}" 54 | svc_YARN = "${var.svc_YARN}" 55 | enable_debug = "${var.enable_debug}" 56 | rangeradmin_password = "${var.rangeradmin_password}" 57 | yarn_scheduler = "${var.yarn_scheduler}" 58 | } 59 | 60 | timeouts { 61 | create = "30m" 62 | } 63 | } 64 | // Block Volume Creation for Utility 65 | 66 | # Log Volume for /var/log/cloudera 67 | resource "oci_core_volume" "UtilLogVolume" { 68 | count = 1 69 | availability_domain = "${var.availability_domain}" 70 | compartment_id = "${var.compartment_ocid}" 71 | display_name = "Cloudera Manager ${format("%01d", count.index+1)} Log Data" 72 | size_in_gbs = "${var.log_volume_size_in_gbs}" 73 | } 74 | 75 | resource "oci_core_volume_attachment" "UtilLogAttachment" { 76 | count = 1 77 | attachment_type = "iscsi" 78 | instance_id = "${oci_core_instance.Utility.id}" 79 | volume_id = "${oci_core_volume.UtilLogVolume.*.id[count.index]}" 80 | device = "/dev/oracleoci/oraclevdb" 81 | } 82 | 83 | # Data Volume for /opt/cloudera 84 | resource "oci_core_volume" "UtilClouderaVolume" { 85 | count = 1 86 | availability_domain = "${var.availability_domain}" 87 | compartment_id = "${var.compartment_ocid}" 88 | display_name = "Cloudera Manager ${format("%01d", count.index+1)} Data" 89 | size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 90 | } 91 | 92 | resource "oci_core_volume_attachment" "UtilClouderaAttachment" { 93 | count = 1 94 | attachment_type = "iscsi" 95 | instance_id = "${oci_core_instance.Utility.id}" 96 | volume_id = "${oci_core_volume.UtilClouderaVolume.*.id[count.index]}" 97 | device = "/dev/oracleoci/oraclevdc" 98 | } 99 | -------------------------------------------------------------------------------- /modules/utility/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cm-url" { value = "http://${data.oci_core_vnic.utility_node_vnic.public_ip_address}:7180/cmf/" } 2 | output "cm-commands-url" { value = "http://${data.oci_core_vnic.utility_node_vnic.public_ip_address}:7180/cmf/commands/commands" } 3 | output "public-ip" { value = "${data.oci_core_vnic.utility_node_vnic.public_ip_address}" } 4 | -------------------------------------------------------------------------------- /modules/utility/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # Environmental variables 3 | # You probably want to define these as environmental variables. 4 | # Instructions on that are here: https://github.com/oci-quickstart/oci-prerequisites 5 | # --------------------------------------------------------------------------------------------------------------------- 6 | 7 | variable "region" {} 8 | variable "compartment_ocid" {} 9 | variable "ssh_public_key" {} 10 | variable "instances" {} 11 | variable "subnet_id" {} 12 | variable "user_data" {} 13 | variable "image_ocid" {} 14 | variable "cm_version" {} 15 | variable "meta_db_type" {} 16 | variable "cloudera_version" {} 17 | variable "worker_shape" {} 18 | variable "block_volume_count" {} 19 | variable "cloudera_manager" {} 20 | variable "cm_install" {} 21 | variable "deploy_on_oci" {} 22 | variable "secure_cluster" {} 23 | variable "hdfs_ha" {} 24 | variable "cluster_name" {} 25 | variable "hide_private_subnet" { 26 | default = "true" 27 | } 28 | variable "cluster_subnet" {} 29 | variable "bastion_subnet" {} 30 | variable "utility_subnet" {} 31 | variable "cm_username" {} 32 | variable "cm_password" {} 33 | variable "vcore_ratio" {} 34 | variable "svc_ATLAS" {} 35 | variable "svc_HBASE" {} 36 | variable "svc_HDFS" {} 37 | variable "svc_HIVE" {} 38 | variable "svc_IMPALA" {} 39 | variable "svc_KAFKA" {} 40 | variable "svc_OOZIE" {} 41 | variable "svc_RANGER" {} 42 | variable "svc_SOLR" {} 43 | variable "svc_SPARK_ON_YARN" {} 44 | variable "svc_SQOOP_CLIENT" {} 45 | variable "svc_YARN" {} 46 | variable "rangeradmin_password" {} 47 | variable "enable_debug" {} 48 | variable "yarn_scheduler" {} 49 | # --------------------------------------------------------------------------------------------------------------------- 50 | # Optional variables 51 | # You can modify these. 52 | # --------------------------------------------------------------------------------------------------------------------- 53 | 54 | variable "availability_domain" { 55 | default = "2" 56 | } 57 | 58 | variable "log_volume_size_in_gbs" { 59 | default = "200" 60 | } 61 | 62 | # Size for Volume across all hosts deployed to /opt/cloudera 63 | 64 | variable "cloudera_volume_size_in_gbs" { 65 | default = "300" 66 | } 67 | 68 | # Size for NameNode and SecondaryNameNode data volume (Journal Data) 69 | 70 | variable "nn_volume_size_in_gbs" { 71 | default = "500" 72 | } 73 | 74 | # 75 | # Set Cluster Shapes in this section 76 | # 77 | 78 | variable "utility_instance_shape" { 79 | default = "VM.Standard2.8" 80 | } 81 | 82 | # --------------------------------------------------------------------------------------------------------------------- 83 | # Constants 84 | # You probably don't need to change these. 85 | # --------------------------------------------------------------------------------------------------------------------- 86 | -------------------------------------------------------------------------------- /modules/worker/main.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_instance" "Worker" { 2 | count = "${var.instances}" 3 | availability_domain = "${var.availability_domain}" 4 | compartment_id = "${var.compartment_ocid}" 5 | shape = "${var.worker_instance_shape}" 6 | display_name = "Cloudera Worker ${format("%01d", count.index+1)}" 7 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 8 | 9 | source_details { 10 | source_type = "image" 11 | source_id = "${var.image_ocid}" 12 | } 13 | 14 | create_vnic_details { 15 | subnet_id = "${var.enable_secondary_vnic ? var.blockvolume_subnet_id : var.subnet_id}" 16 | display_name = "Cloudera Worker ${format("%01d", count.index+1)}" 17 | hostname_label = "${var.enable_secondary_vnic ? data.null_data_source.hostname_labels[count.index].outputs["secondary_label"] : data.null_data_source.hostname_labels[count.index].outputs["primary_label"]}" 18 | assign_public_ip = "${var.hide_public_subnet ? false : true}" 19 | } 20 | 21 | metadata = { 22 | ssh_authorized_keys = "${var.ssh_public_key}" 23 | user_data = "${var.user_data}" 24 | cloudera_manager = "${var.cloudera_manager}" 25 | cloudera_version = "${var.cloudera_version}" 26 | cm_version = "${var.cm_version}" 27 | block_volume_count = "${var.block_volume_count}" 28 | objectstoreRAID = "${var.objectstoreRAID}" 29 | enable_secondary_vnic = "${var.enable_secondary_vnic}" 30 | agent_hostname = "cloudera-worker-${count.index+1}.${var.worker_domain}" 31 | } 32 | 33 | timeouts { 34 | create = "30m" 35 | } 36 | } 37 | 38 | data "null_data_source" "hostname_labels" { 39 | count = "${var.instances}" 40 | inputs = { 41 | primary_label = "Cloudera-Worker-${format("%01d", count.index+1)}" 42 | secondary_label = "Cloudera-Worker-primary-${format("%01d", count.index+1)}" 43 | } 44 | } 45 | 46 | data "oci_core_vnic" "secondary_vnic" { 47 | count = "${var.secondary_vnic_count * var.instances}" 48 | vnic_id = "${element(oci_core_vnic_attachment.secondary_vnic_attachment.*.vnic_id, count.index)}" 49 | } 50 | 51 | resource "oci_core_vnic_attachment" "secondary_vnic_attachment" { 52 | count = "${var.secondary_vnic_count * var.instances}" 53 | instance_id = "${oci_core_instance.Worker[count.index].id}" 54 | display_name = "SecondaryVnicAttachment_${count.index}" 55 | 56 | create_vnic_details { 57 | subnet_id = "${var.subnet_id}" 58 | display_name = "SecondaryVnic_${count.index}" 59 | assign_public_ip = "${var.hide_public_subnet ? false : true}" 60 | hostname_label = "Cloudera-Worker-${format("%01d", count.index+1)}" 61 | } 62 | nic_index = "1" 63 | } 64 | 65 | // Block Volume Creation for Worker 66 | 67 | # Log Volume for /var/log/cloudera 68 | resource "oci_core_volume" "WorkerLogVolume" { 69 | count = "${var.instances}" 70 | availability_domain = "${var.availability_domain}" 71 | compartment_id = "${var.compartment_ocid}" 72 | display_name = "Cloudera Worker ${format("%01d", count.index+1)} Log Data" 73 | size_in_gbs = "${var.log_volume_size_in_gbs}" 74 | } 75 | 76 | resource "oci_core_volume_attachment" "WorkerLogAttachment" { 77 | count = "${var.instances}" 78 | attachment_type = "iscsi" 79 | instance_id = "${oci_core_instance.Worker[count.index].id}" 80 | volume_id = "${oci_core_volume.WorkerLogVolume[count.index].id}" 81 | device = "/dev/oracleoci/oraclevdb" 82 | } 83 | 84 | # Data Volume for /opt/cloudera 85 | resource "oci_core_volume" "WorkerClouderaVolume" { 86 | count = "${var.instances}" 87 | availability_domain = "${var.availability_domain}" 88 | compartment_id = "${var.compartment_ocid}" 89 | display_name = "Cloudera Worker ${format("%01d", count.index+1)} Cloudera Data" 90 | size_in_gbs = "${var.cloudera_volume_size_in_gbs}" 91 | } 92 | 93 | resource "oci_core_volume_attachment" "WorkerClouderaAttachment" { 94 | count = "${var.instances}" 95 | attachment_type = "iscsi" 96 | instance_id = "${oci_core_instance.Worker[count.index].id}" 97 | volume_id = "${oci_core_volume.WorkerClouderaVolume[count.index].id}" 98 | device = "/dev/oracleoci/oraclevdc" 99 | } 100 | 101 | # RAID Volumes for Object Store Cache 102 | resource "oci_core_volume" "WorkerRAIDVolume" { 103 | count = var.objectstoreRAID ? (var.instances * 4) : 0 104 | availability_domain = "${var.availability_domain}" 105 | compartment_id = "${var.compartment_ocid}" 106 | display_name = "Cloudera Worker ${format("%01d", floor((count.index / 4)+1))} RAID ${format("%01d", floor((count.index%(4))+1))}" 107 | size_in_gbs = 700 108 | } 109 | 110 | resource "oci_core_volume_attachment" "WorkerRAIDAttachment" { 111 | count = var.objectstoreRAID ? (var.instances * 4) : 0 112 | attachment_type = "iscsi" 113 | instance_id = "${oci_core_instance.Worker[floor(count.index/4)].id}" 114 | volume_id = "${oci_core_volume.WorkerRAIDVolume[count.index].id}" 115 | device = "${var.data_volume_attachment_device[floor(count.index%(4))]}" 116 | } 117 | 118 | # Data Volumes for HDFS 119 | resource "oci_core_volume" "WorkerDataVolume" { 120 | count = "${(var.instances * var.block_volumes_per_worker)}" 121 | availability_domain = "${var.availability_domain}" 122 | compartment_id = "${var.compartment_ocid}" 123 | display_name = "Cloudera Worker ${format("%01d", floor((count.index / var.block_volumes_per_worker)+1))} HDFS Data ${format("%01d", floor((count.index%(var.block_volumes_per_worker))+1))}" 124 | size_in_gbs = "${var.data_blocksize_in_gbs}" 125 | vpus_per_gb = "${var.vpus_per_gb}" 126 | } 127 | 128 | resource "oci_core_volume_attachment" "WorkerDataAttachment" { 129 | count = "${(var.instances * var.block_volumes_per_worker)}" 130 | attachment_type = "iscsi" 131 | instance_id = "${oci_core_instance.Worker[floor(count.index/var.block_volumes_per_worker)].id}" 132 | volume_id = "${oci_core_volume.WorkerDataVolume[count.index].id}" 133 | device = "${var.objectstoreRAID ? var.data_volume_attachment_device[(floor(count.index%(var.block_volumes_per_worker))+4)] : var.data_volume_attachment_device[floor(count.index%(var.block_volumes_per_worker))]}" 134 | } 135 | 136 | -------------------------------------------------------------------------------- /modules/worker/outputs.tf: -------------------------------------------------------------------------------- 1 | output "block-volume-count" { value = "${var.block_volumes_per_worker}" } 2 | output "block-volume-size" { value = "${var.data_blocksize_in_gbs}" } 3 | -------------------------------------------------------------------------------- /modules/worker/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # Environmental variables 3 | # You probably want to define these as environmental variables. 4 | # Instructions on that are here: https://github.com/oci-quickstart/oci-prerequisites 5 | # --------------------------------------------------------------------------------------------------------------------- 6 | 7 | variable "region" {} 8 | variable "compartment_ocid" {} 9 | variable "ssh_public_key" {} 10 | variable "instances" {} 11 | variable "subnet_id" {} 12 | variable "blockvolume_subnet_id" {} 13 | variable "user_data" {} 14 | variable "image_ocid" {} 15 | variable "cm_version" {} 16 | variable "cloudera_version" {} 17 | variable "cloudera_manager" {} 18 | variable "block_volume_count" {} 19 | variable "hide_public_subnet" { 20 | default = "true" 21 | } 22 | variable "objectstoreRAID" { 23 | default = "false" 24 | } 25 | variable "secondary_vnic_count" { 26 | default = "0" 27 | } 28 | variable "enable_secondary_vnic" { 29 | default = "false" 30 | } 31 | variable "worker_domain" {} 32 | # --------------------------------------------------------------------------------------------------------------------- 33 | # Optional variables 34 | # You can modify these. 35 | # --------------------------------------------------------------------------------------------------------------------- 36 | 37 | variable "availability_domain" { 38 | default = "2" 39 | } 40 | 41 | # Number of Workers in the Cluster 42 | 43 | variable "worker_node_count" { 44 | default = "5" 45 | } 46 | 47 | variable "data_blocksize_in_gbs" { 48 | default = "700" 49 | } 50 | 51 | variable "block_volumes_per_worker" {} 52 | 53 | variable "vpus_per_gb" { 54 | default = "10" 55 | } 56 | 57 | # Size for Cloudera Log Volumes across all hosts deployed to /var/log/cloudera 58 | 59 | variable "log_volume_size_in_gbs" { 60 | default = "200" 61 | } 62 | 63 | # Size for Volume across all hosts deployed to /opt/cloudera 64 | 65 | variable "cloudera_volume_size_in_gbs" { 66 | default = "300" 67 | } 68 | 69 | # Size for NameNode and SecondaryNameNode data volume (Journal Data) 70 | 71 | variable "nn_volume_size_in_gbs" { 72 | default = "500" 73 | } 74 | 75 | # 76 | # Set Cluster Shapes in this section 77 | # 78 | 79 | variable "worker_instance_shape" { 80 | default = "BM.DenseIO2.52" 81 | } 82 | 83 | 84 | # --------------------------------------------------------------------------------------------------------------------- 85 | # Constants 86 | # You probably don't need to change these. 87 | # --------------------------------------------------------------------------------------------------------------------- 88 | 89 | // Volume Mapping - used to map Worker Block Volumes consistently to the OS 90 | variable "data_volume_attachment_device" { 91 | type = "map" 92 | default = { 93 | "0" = "/dev/oracleoci/oraclevdd" 94 | "1" = "/dev/oracleoci/oraclevde" 95 | "2" = "/dev/oracleoci/oraclevdf" 96 | "3" = "/dev/oracleoci/oraclevdg" 97 | "4" = "/dev/oracleoci/oraclevdh" 98 | "5" = "/dev/oracleoci/oraclevdi" 99 | "6" = "/dev/oracleoci/oraclevdj" 100 | "7" = "/dev/oracleoci/oraclevdk" 101 | "8" = "/dev/oracleoci/oraclevdl" 102 | "9" = "/dev/oracleoci/oraclevdm" 103 | "10" = "/dev/oracleoci/oraclevdn" 104 | "11" = "/dev/oracleoci/oraclevdo" 105 | "12" = "/dev/oracleoci/oraclevdp" 106 | "13" = "/dev/oracleoci/oraclevdq" 107 | "14" = "/dev/oracleoci/oraclevdr" 108 | "15" = "/dev/oracleoci/oraclevds" 109 | "16" = "/dev/oracleoci/oraclevdt" 110 | "17" = "/dev/oracleoci/oraclevdu" 111 | "18" = "/dev/oracleoci/oraclevdv" 112 | "19" = "/dev/oracleoci/oraclevdw" 113 | "20" = "/dev/oracleoci/oraclevdx" 114 | "21" = "/dev/oracleoci/oraclevdy" 115 | "22" = "/dev/oracleoci/oraclevdz" 116 | "23" = "/dev/oracleoci/oraclevdab" 117 | "24" = "/dev/oracleoci/oraclevdac" 118 | "25" = "/dev/oracleoci/oraclevdad" 119 | "26" = "/dev/oracleoci/oraclevdae" 120 | "27" = "/dev/oracleoci/oraclevdaf" 121 | "28" = "/dev/oracleoci/oraclevdag" 122 | } 123 | } 124 | 125 | -------------------------------------------------------------------------------- /network.tf: -------------------------------------------------------------------------------- 1 | # All regions filtered down to current region 2 | # Example return value [ {"key" = "IAD", "name" = "us-ashburn-1" } ] 3 | data "oci_identity_regions" "filtered_regions" { 4 | filter { 5 | name = "name" 6 | values = [var.region] 7 | } 8 | } 9 | 10 | locals { 11 | # Construct gateway string from 3 char name of current region 12 | oci_service_gateway = "all-${lower(data.oci_identity_regions.filtered_regions.regions[0].key)}-services-in-oracle-services-network" 13 | } 14 | 15 | module "network" { 16 | source = "./modules/network" 17 | tenancy_ocid = "${var.tenancy_ocid}" 18 | compartment_ocid = "${var.compartment_ocid}" 19 | availability_domain = "${var.availability_domain}" 20 | region = "${var.region}" 21 | oci_service_gateway = "${local.oci_service_gateway}" 22 | useExistingVcn = "${var.useExistingVcn}" 23 | custom_cidrs = "${var.custom_cidrs}" 24 | VPC_CIDR = "${var.VPC_CIDR}" 25 | edge_cidr = "${var.edge_cidr}" 26 | public_cidr = "${var.public_cidr}" 27 | private_cidr = "${var.private_cidr}" 28 | blockvolume_cidr = "${var.blockvolume_cidr}" 29 | custom_vcn = ["${var.myVcn}"] 30 | enable_secondary_vnic = "${var.enable_secondary_vnic}" 31 | clusterSubnet = "${var.clusterSubnet}" 32 | bastionSubnet = "${var.bastionSubnet}" 33 | utilitySubnet = "${var.utilitySubnet}" 34 | blockvolumeSubnet = "${var.blockvolumeSubnet}" 35 | myVcn = "${var.myVcn}" 36 | } 37 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | output "CLOUDERA_INFO_README" { value = "Cluster Builds can take anywhere from 15 to 30 minutes depending on deployment options. It is recommended to wait at least 15 minutes before logging into Cloudera Manager, and do not perform any setup activities through the UI. Deployment progress can be checked on the Utility host by 'sudo tail -f /var/log/cloudera-OCI-initialize.log'" } 2 | output "CLOUDERA_MANAGER" { value = "http://${module.utility.public-ip}:7180/cmf/" } 3 | output "CLOUDERA_MANAGER_LOGIN" { value = "User: ${var.cm_username} Password: ${var.cm_password}" } 4 | output "SSH_KEY_INFO" { value = "${var.provide_ssh_key ? "SSH Key Provided by user" : "See below for generated SSH private key."}" } 5 | output "SSH_PRIVATE_KEY" { value = "${var.provide_ssh_key ? "SSH Key Provided by user" : tls_private_key.key.private_key_pem}" } 6 | -------------------------------------------------------------------------------- /schema.yaml: -------------------------------------------------------------------------------- 1 | title: Sample input variable schema 2 | schemaVersion: 1.1.0 3 | version: "2020311" 4 | locale: "en" 5 | groupings: 6 | - title: "SSH Key" 7 | variables: 8 | - ${provide_ssh_key} 9 | - ${ssh_provided_key} 10 | - title: "Availabilty Domain" 11 | variables: 12 | - ${availability_domain} 13 | - title: "VCN Options" 14 | variables: 15 | - ${useExistingVcn} 16 | - ${myVcn} 17 | - ${hide_public_subnet} 18 | - ${hide_private_subnet} 19 | - ${vcn_dns_label} 20 | - ${custom_cidrs} 21 | - ${VPC_CIDR} 22 | - ${edge_cidr} 23 | - ${public_cidr} 24 | - ${private_cidr} 25 | - title: "Cloudera Options" 26 | variables: 27 | - ${cm_username} 28 | - ${cm_password} 29 | - ${cm_version} 30 | - ${cloudera_version} 31 | - ${cluster_name} 32 | - ${secure_cluster} 33 | - ${hdfs_ha} 34 | - ${AdvancedOptions} 35 | - ${enable_debug} 36 | - ${log_volume_size_in_gbs} 37 | - ${cloudera_volume_size_in_gbs} 38 | - ${svc_HBASE} 39 | - ${svc_HDFS} 40 | - ${svc_HIVE} 41 | - ${svc_IMPALA} 42 | - ${svc_KAFKA} 43 | - ${svc_OOZIE} 44 | - ${svc_SOLR} 45 | - ${svc_SPARK_ON_YARN} 46 | - ${svc_SQOOP_CLIENT} 47 | - ${svc_YARN} 48 | - ${vcore_ratio} 49 | - ${yarn_scheduler} 50 | - title: "Advanced CDP Options" 51 | variables: 52 | - ${svc_ATLAS} 53 | - ${svc_RANGER} 54 | - ${rangeradmin_password} 55 | visible: 56 | and: 57 | - ${AdvancedOptions} 58 | - and: 59 | - eq: 60 | - ${cloudera_version} 61 | - "7.1.7.0" 62 | - and: 63 | - not: 64 | - eq: 65 | - ${cloudera_version} 66 | - "5.10.2.5" 67 | - not: 68 | - eq: 69 | - ${cloudera_version} 70 | - "5.11.2.4" 71 | - not: 72 | - eq: 73 | - ${cloudera_version} 74 | - "5.12.2.4" 75 | - not: 76 | - eq: 77 | - ${cloudera_version} 78 | - "5.13.3.2" 79 | - not: 80 | - eq: 81 | - ${cloudera_version} 82 | - "5.14.4.3" 83 | - not: 84 | - eq: 85 | - ${cloudera_version} 86 | - "5.15.2.3" 87 | - not: 88 | - eq: 89 | - ${cloudera_version} 90 | - "5.16.2.8" 91 | - not: 92 | - eq: 93 | - ${cloudera_version} 94 | - "6.0.0" 95 | - not: 96 | - eq: 97 | - ${cloudera_version} 98 | - "6.0.1" 99 | - not: 100 | - eq: 101 | - ${cloudera_version} 102 | - "6.1.0" 103 | - not: 104 | - eq: 105 | - ${cloudera_version} 106 | - "6.1.1" 107 | - not: 108 | - eq: 109 | - ${cloudera_version} 110 | - "6.2.0" 111 | - not: 112 | - eq: 113 | - ${cloudera_version} 114 | - "6.2.1" 115 | - not: 116 | - eq: 117 | - ${cloudera_version} 118 | - "6.3.2" 119 | - title: "Worker Node Options" 120 | variables: 121 | - ${worker_instance_shape} 122 | - ${worker_node_count} 123 | - ${objectstoreRAID} 124 | - ${enable_block_volumes} 125 | - ${block_volumes_per_worker} 126 | - ${data_blocksize_in_gbs} 127 | - ${customize_block_volume_performance} 128 | - ${block_volume_high_performance} 129 | - ${block_volume_cost_savings} 130 | - ${clusterSubnet} 131 | - ${enable_secondary_vnic} 132 | - ${blockvolume_cidr} 133 | - ${blockvolumeSubnet} 134 | - title: "Master Node Options" 135 | variables: 136 | - ${master_instance_shape} 137 | - ${master_node_count} 138 | - ${nn_volume_size_in_gbs} 139 | - title: "Utility Node Options" 140 | variables: 141 | - ${utility_instance_shape} 142 | - ${utilitySubnet} 143 | - ${meta_db_type} 144 | - title: "Edge Node Options" 145 | variables: 146 | - ${use_edge_nodes} 147 | - ${bastion_instance_shape} 148 | - ${bastion_node_count} 149 | - ${bastionSubnet} 150 | - title: "Pre-Defined" 151 | variables: 152 | - ${region} 153 | - ${compartment_ocid} 154 | - ${tenancy_ocid} 155 | - ${CentOSImageOCID} 156 | - ${OELImageOCID} 157 | - ${oci_service_gateway} 158 | - ${AD} 159 | - ${vpus_per_gb} 160 | - ${secondary_vnic_count} 161 | - ${blockvolume_subnet_id} 162 | - ${worker_domain} 163 | visible: false 164 | 165 | variables: 166 | cm_version: 167 | type: enum 168 | enum: 169 | - "6.0.0" 170 | - "6.0.1" 171 | - "6.1.0" 172 | - "6.2.0" 173 | - "6.3.0" 174 | - "6.3.1" 175 | - "7.0.3" 176 | - "7.1.1" 177 | - "7.1.2" 178 | - "7.1.3" 179 | - "7.1.4" 180 | - "7.2.4" 181 | - "7.2.6" 182 | - "7.3.1" 183 | - "7.4.4" 184 | - "7.5.1" 185 | title: "Cloudera Manager Version" 186 | description: "Choose the version of Cloudera Manager to deploy, CDP requires 7.1.4" 187 | required: true 188 | default: "7.4.4" 189 | 190 | cloudera_version: 191 | type: enum 192 | enum: 193 | - "5.10.2.5" 194 | - "5.11.2.4" 195 | - "5.12.2.4" 196 | - "5.13.3.2" 197 | - "5.14.4.3" 198 | - "5.15.2.3" 199 | - "5.16.2.8" 200 | - "6.0.0" 201 | - "6.0.1" 202 | - "6.1.0" 203 | - "6.1.1" 204 | - "6.2.0" 205 | - "6.2.1" 206 | - "6.3.2" 207 | - "7.1.7.0" 208 | title: "Cluster Version" 209 | description: "Choose the version of CDH to deploy, for CDP choose 7.1.4.0" 210 | required: true 211 | default: "7.1.7.0" 212 | 213 | cluster_name: 214 | type: string 215 | title: "Cluster Name" 216 | description: "Name your Cluster" 217 | required: true 218 | 219 | useExistingVcn: 220 | type: boolean 221 | title: "Use Existing VCN" 222 | description: "Click to use existing VCN, otherwise VCN and Subnets will be created" 223 | required: true 224 | default: false 225 | 226 | myVcn: 227 | type: oci:core:vcn:id 228 | title: "Existing VCN" 229 | description: "Select Existing VCN" 230 | dependsOn: 231 | compartmentId: ${compartment_ocid} 232 | visible: ${useExistingVcn} 233 | required: true 234 | 235 | custom_cidrs: 236 | type: boolean 237 | title: "Customize Network CIDRS" 238 | description: "Click to customize CIDR ranges, only applicable when creating VCN as part of deployment" 239 | required: true 240 | default: false 241 | 242 | utilitySubnet: 243 | type: oci:core:subnet:id 244 | title: "Utility Subnet" 245 | description: "Select Subnet - Ensure the Subnet is in the same Availability Domain selected above" 246 | dependsOn: 247 | compartmentId: ${compartment_ocid} 248 | vcnId: ${myVcn} 249 | hidePrivateSubnet: ${hide_private_subnet} 250 | visible: ${useExistingVcn} 251 | required: true 252 | 253 | clusterSubnet: 254 | type: oci:core:subnet:id 255 | title: "Cluster Subnet" 256 | description: "Select Subnet - Ensure the Subnet is in the same Availability Domain selected above" 257 | dependsOn: 258 | compartmentId: ${compartment_ocid} 259 | vcnId: ${myVcn} 260 | hidePublicSubnet: ${hide_public_subnet} 261 | visible: ${useExistingVcn} 262 | required: true 263 | 264 | bastionSubnet: 265 | type: oci:core:subnet:id 266 | title: "Utility Subnet" 267 | description: "Select Subnet - Ensure the Subnet is in the same Availability Domain selected above" 268 | dependsOn: 269 | compartmentId: ${compartment_ocid} 270 | vcnId: ${myVcn} 271 | hidePrivateSubnet: ${hide_private_subnet} 272 | visible: ${useExistingVcn} 273 | required: true 274 | 275 | blockvolumeSubnet: 276 | type: oci:core:subnet:id 277 | title: "BlockVolume Subnet" 278 | description: "Select Subnet - Ensure the Subnet is in the same Availability Domain selected above" 279 | dependsOn: 280 | compartmentId: ${compartment_ocid} 281 | vcnId: ${myVcn} 282 | hidePublicSubnet: ${hide_public_subnet} 283 | visible: 284 | and: 285 | - ${useExistingVcn} 286 | - ${enable_secondary_vnic} 287 | required: true 288 | 289 | hide_private_subnet: 290 | type: boolean 291 | title: "Deploy Utility & Edge hosts to Public Networks" 292 | description: "If you wish to deploy to private networks and use VPN, un-check this" 293 | default: true 294 | visible: ${useExistingVcn} 295 | 296 | hide_public_subnet: 297 | type: boolean 298 | title: "Deploy Cluster to Private Network Only" 299 | description: "This is highly suggested, disable at your own risk" 300 | default: true 301 | visible: ${useExistingVcn} 302 | 303 | availability_domain: 304 | type: oci:identity:availabilitydomain:name 305 | title: "Availability Domain" 306 | description: "Select AD" 307 | dependsOn: 308 | compartmentId: ${compartment_ocid} 309 | required: true 310 | 311 | secure_cluster: 312 | type: Boolean 313 | title: "Kerberos Security" 314 | description: "Click to Enable Kerberos for Secure Cluster" 315 | required: true 316 | 317 | hdfs_ha: 318 | type: boolean 319 | title: "High Availability" 320 | description: "Click to Enable HDFS High Availability" 321 | required: true 322 | 323 | worker_instance_shape: 324 | type: enum 325 | enum: 326 | - "VM.Standard2.8" 327 | - "VM.Standard2.16" 328 | - "VM.Standard2.24" 329 | - "BM.Standard2.52" 330 | - "BM.Standard.E2.64" 331 | - "VM.DenseIO2.8" 332 | - "VM.DenseIO2.16" 333 | - "VM.DenseIO2.24" 334 | - "BM.DenseIO2.52" 335 | - "BM.HPC2.36" 336 | title: "Shape of Worker Nodes" 337 | required: true 338 | 339 | worker_node_count: 340 | type: integer 341 | minimum: 3 342 | title: "Number of Worker Nodes" 343 | description: "3 is the minimum requirement" 344 | required: true 345 | 346 | master_instance_shape: 347 | type: enum 348 | enum: 349 | - "VM.Standard2.8" 350 | - "VM.Standard2.16" 351 | - "VM.Standard2.24" 352 | - "BM.Standard2.52" 353 | - "BM.Standard.E2.64" 354 | - "VM.DenseIO2.8" 355 | - "VM.DenseIO2.16" 356 | - "VM.DenseIO2.24" 357 | - "BM.DenseIO2.52" 358 | - "BM.HPC2.36" 359 | title: "Shape of Master Nodes" 360 | required: true 361 | 362 | master_node_count: 363 | type: integer 364 | minimum: 2 365 | title: "Number of Master Nodes" 366 | description: "2 is the minimum requirement" 367 | required: true 368 | 369 | utility_instance_shape: 370 | type: enum 371 | enum: 372 | - "VM.Standard2.8" 373 | - "VM.Standard2.16" 374 | - "VM.Standard2.24" 375 | - "BM.Standard2.52" 376 | - "BM.Standard.E2.64" 377 | - "VM.DenseIO2.8" 378 | - "VM.DenseIO2.16" 379 | - "VM.DenseIO2.24" 380 | - "BM.DenseIO2.52" 381 | - "BM.HPC2.36" 382 | title: "Shape of Utility Node" 383 | required: true 384 | 385 | bastion_instance_shape: 386 | type: enum 387 | enum: 388 | - "VM.Standard2.1" 389 | - "VM.Standard.E2.1" 390 | - "VM.Standard2.2" 391 | - "VM.Standard.E2.2" 392 | - "VM.Standard2.4" 393 | - "VM.Standard.E2.4" 394 | - "VM.Standard2.8" 395 | - "VM.Standard.E2.8" 396 | - "VM.Standard2.16" 397 | - "VM.Standard2.24" 398 | - "BM.Standard2.52" 399 | - "VM.DenseIO2.8" 400 | - "VM.DenseIO2.16" 401 | - "VM.DenseIO2.24" 402 | - "BM.DenseIO2.52" 403 | title: "Shape of Edge Nodes" 404 | required: true 405 | visible: ${use_edge_nodes} 406 | 407 | bastion_node_count: 408 | type: integer 409 | title: "Number of Edge Nodes" 410 | description: "Enter a number, 0 to service limit of shape" 411 | minimum: 0 412 | required: true 413 | visible: ${use_edge_nodes} 414 | 415 | ssh_provided_key: 416 | type: string 417 | title: "SSH Public Key" 418 | description: "Copy/Paste the contents of your SSH Public Key" 419 | required: true 420 | default: "" 421 | visible: ${provide_ssh_key} 422 | 423 | data_blocksize_in_gbs: 424 | type: integer 425 | title: "HDFS Block Volume Size ( GB )" 426 | description: "700 to 32,768 " 427 | minimum: 700 428 | maximum: 32768 429 | required: true 430 | visible: ${enable_block_volumes} 431 | 432 | block_volumes_per_worker: 433 | type: integer 434 | title: "Number of Block Volumes for HDFS" 435 | description: "0 to 29" 436 | minimum: 0 437 | maximum: 29 438 | required: true 439 | visible: ${enable_block_volumes} 440 | 441 | customize_block_volume_performance: 442 | type: boolean 443 | title: "Custom Block Volume Performance" 444 | description: "Click to customize Block Volume performance. Default profile is Balanced. When enabled, only select one sub-option." 445 | default: false 446 | visible: ${enable_block_volumes} 447 | 448 | block_volume_high_performance: 449 | type: boolean 450 | title: "High Performance Block Volumes" 451 | description: "Click to enable High Performance for HDFS Block Volumes. This comes at a higher cost per GB." 452 | default: false 453 | visible: 454 | and: 455 | - ${enable_block_volumes} 456 | - ${customize_block_volume_performance} 457 | 458 | block_volume_cost_savings: 459 | type: boolean 460 | title: "Lower Cost Block Volumes" 461 | description: "Click to enable Lower Cost for HDFS Block Volumes. This lowers performance for cost savings per GB." 462 | default: false 463 | visible: 464 | and: 465 | - ${enable_block_volumes} 466 | - ${customize_block_volume_performance} 467 | 468 | nn_volume_size_in_gbs: 469 | type: integer 470 | title: "Block Volume Size for NameNode Metadata" 471 | description: "500 to 32,768" 472 | minimum: 500 473 | maximum: 32768 474 | required: true 475 | visible: ${enable_block_volumes} 476 | 477 | log_volume_size_in_gbs: 478 | type: integer 479 | title: "Volume size in GB for CDH Logs" 480 | description: "50 to 32,768" 481 | minimum: 50 482 | maximum: 32768 483 | required: true 484 | visible: ${AdvancedOptions} 485 | 486 | cloudera_volume_size_in_gbs: 487 | type: integer 488 | title: "Volume size in GB for CDH Data (Parcels)" 489 | description: "150 to 32,768" 490 | minimum: 150 491 | maximum: 32768 492 | required: true 493 | visible: ${AdvancedOptions} 494 | 495 | VPC_CIDR: 496 | type: string 497 | title: "VPC CIDR for VCN" 498 | description: "Customize VCN top level CIDR" 499 | visible: ${custom_cidrs} 500 | 501 | edge_cidr: 502 | type: string 503 | title: "Edge Subnet CIDR" 504 | description: "Customize Edge Subnet CIDR, ensure this fits in VCN CIDR range." 505 | visible: ${custom_cidrs} 506 | 507 | public_cidr: 508 | type: string 509 | title: "Public Subnet CIDR" 510 | description: "Customize Public Subnet CIDR, ensure this fits in VCN CIDR range." 511 | visible: ${custom_cidrs} 512 | 513 | private_cidr: 514 | type: string 515 | title: "Private Subnet CIDR" 516 | description: "Customize Private Subnet CIDR, ensure this fits in VCN CIDR range." 517 | visible: ${custom_cidrs} 518 | 519 | blockvolume_cidr: 520 | type: string 521 | title: "BlockVolume Subnet CIDR" 522 | description: "Customize BlockVolume Subnet CIDR, ensure this fits in VCN CIDR range. This subnet is used on BM hosts to segregate storage network traffic from cluster traffic." 523 | visible: 524 | and: 525 | - ${custom_cidrs} 526 | - ${enable_secondary_vnic} 527 | 528 | vcn_dns_label: 529 | type: string 530 | title: "VCN DNS Label" 531 | description: "Set the VCN DNS label to be used when creating VCN. Default is 'clouderavcn' which sets the VCN domain to 'clouderavcn.oraclevcn.com'" 532 | 533 | objectstoreRAID: 534 | type: boolean 535 | title: "RAID0 Block Volume Cache" 536 | description: "Enable RAID0 Block Volume cache to enhance througput when moving data to/from Object Storage. This is recommended when using DistCP." 537 | default: false 538 | 539 | meta_db_type: 540 | type: enum 541 | title: "Cloudera Manager Database" 542 | description: "Pick which database to use for Cloudera Manager. Note that Postgresql is not recommended for production clusters." 543 | enum: 544 | - "mysql" 545 | - "postgres" 546 | required: true 547 | visible: ${AdvancedOptions} 548 | 549 | use_edge_nodes: 550 | type: boolean 551 | title: "Enable Edge Nodes" 552 | description: "Check to enable Edge Nodes for the cluster." 553 | default: false 554 | 555 | enable_block_volumes: 556 | type: boolean 557 | title: "Enable Block Volumes for HDFS" 558 | description: "Check to enable Block Volumes for use with HDFS. This is optional for BM.Dense hosts, required for all others." 559 | default: true 560 | 561 | cm_username: 562 | type: string 563 | title: "Cloudera Manager Admin" 564 | description: "Set the admin username, it must be something other than the default." 565 | required: true 566 | 567 | cm_password: 568 | type: password 569 | title: "Cloudera Manager Admin Password" 570 | description: "Set the Admin user default password, do not use spaces as this password is not sanitized. This will be used for deployment, it is encouraged you change this after deployment is complete." 571 | required: true 572 | 573 | provide_ssh_key: 574 | type: boolean 575 | title: "Provide SSH Key" 576 | description: "Un-Check to generate SSH key as part of deployment process. This is NOT recommended for persistent environments, you should provide your own key for any production deployment." 577 | 578 | vcore_ratio: 579 | type: int 580 | title: "YARN VCore Ratio" 581 | description: "Set the YARN VCore ratio. This is typically between 1-4. VMs benefit from a smaller VCore ratio, BMs benefit from a higher VCore ratio." 582 | min: 1 583 | max: 4 584 | default: 2 585 | visible: ${AdvancedOptions} 586 | 587 | AdvancedOptions: 588 | type: boolean 589 | title: "Advanced Options" 590 | description: "Enable Advanced Cluster Configuration Options. For advanced users only, changing service parameters may have a negative impact on cluster deployment, modify at your own risk. These parameters only affect initial cluster build, once the cluster is deployed Cloudera Manager must be used for service administration." 591 | 592 | svc_ATLAS: 593 | type: boolean 594 | title: "Atlas Service" 595 | description: "Enable Atlas Service, CDP ONLY" 596 | 597 | svc_HBASE: 598 | type: boolean 599 | title: "HBase Service" 600 | description: "Enable HBase Service" 601 | visible: ${AdvancedOptions} 602 | 603 | svc_HDFS: 604 | type: boolean 605 | title: "HDFS Service" 606 | description: "Enable HDFS Service" 607 | visible: ${AdvancedOptions} 608 | 609 | svc_HIVE: 610 | type: boolean 611 | title: "Hive Service" 612 | description: "Enable Hive Service" 613 | visible: ${AdvancedOptions} 614 | 615 | svc_IMPALA: 616 | type: boolean 617 | title: "Impala Service" 618 | description: "Enable Impala Service" 619 | visible: ${AdvancedOptions} 620 | 621 | svc_KAFKA: 622 | type: boolean 623 | title: "Kafka Service" 624 | description: "Enable Kafa Service" 625 | visible: ${AdvancedOptions} 626 | 627 | svc_OOZIE: 628 | type: boolean 629 | title: "Oozie Service" 630 | description: "Enable Oozie Service" 631 | visible: ${AdvancedOptions} 632 | 633 | svc_RANGER: 634 | type: boolean 635 | title: "Ranger Service" 636 | description: "Enable Ranger Service, CDP ONLY" 637 | 638 | rangeradmin_password: 639 | type: string 640 | title: "Password for Ranger" 641 | description: "Sets default deployment password for rangeradmin, usersync, tagsync, and keystore. Password requires upper case, numbers and special character, if this is not set properly then deployment may fail." 642 | visible: ${svc_RANGER} 643 | 644 | svc_SOLR: 645 | type: boolean 646 | title: "SOLR Service" 647 | description: "Enable SOLR Service" 648 | visible: ${AdvancedOptions} 649 | 650 | svc_SPARK_ON_YARN: 651 | type: boolean 652 | title: "Spark on YARN Service" 653 | description: "Enable Spark on YARN Service" 654 | visible: ${AdvancedOptions} 655 | 656 | svc_SQOOP_CLIENT: 657 | type: boolean 658 | title: "Sqoop Client" 659 | description: "Enable Sqoop Client" 660 | visible: ${AdvancedOptions} 661 | 662 | svc_YARN: 663 | type: boolean 664 | title: "YARN Service" 665 | description: "Enable YARN Service" 666 | visible: ${AdvancedOptions} 667 | 668 | enable_debug: 669 | type: boolean 670 | title: "Enable debug Output" 671 | description: "Enable debug output for python deployment script. Output is found in /var/log/cloudera-OCI-initialize.log" 672 | visible: ${AdvancedOptions} 673 | 674 | yarn_scheduler: 675 | type: enum 676 | enum: 677 | - "fair" 678 | - "fifo" 679 | - "capacity" 680 | title: "YARN Scheduler" 681 | description: "Select YARN Scheduler type" 682 | visible: ${AdvancedOptions} 683 | default: "capacity" 684 | 685 | enable_secondary_vnic: 686 | type: boolean 687 | title: "Enable Secondary VNIC" 688 | description: "Enable a second VNIC on the second physical interface for BM hosts. This will be used for cluster traffic, allowing the primary interface to be used for storage." 689 | visible: 690 | or: 691 | - or: 692 | - eq: 693 | - ${worker_instance_shape} 694 | - "BM.Standard2.52" 695 | - eq: 696 | - ${worker_instance_shape} 697 | - "BM.DenseIO2.52" 698 | - or: 699 | - eq: 700 | - ${worker_instance_shape} 701 | - "BM.HPC2.36" 702 | - eq: 703 | - ${worker_instance_shape} 704 | - "BM.Standard.E2.64" 705 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | # scripts 2 | All scripts in this location are referenced for deployment automation 3 | 4 | * boot.sh is invoked by CloudInit on each instance creation via Terraform. It contains steps which perform inital bootstrapping of the instance prior to provisioning. 5 | * cloudera_manager_boot.sh is a top level boot script for Cloudera Manager (Utility) instance. This is required because subsequent scripts are too large to fit in metadata without compression. 6 | * cms_mysql.sh is invoked by cloudinit on the Utility node to stand up Cloudera Manager and Pre-requisites using MySQL for Metadata. It is compressed and loaded into extended metadata. 7 | * cms_postgres.sh is an older installaltion method using Postgres instead of MySQL for cluster metadata. This is depracated. 8 | * deploy_on_oci.py is the primary Python script invoked to deploy Cloudera EDH v6 using cm_client python libraries. It is compressed and loaded into extended metdata. 9 | 10 | # CloudInit boot scripts 11 | 12 | With the introduction of local KDC for secure cluster, this requires some setup at the instance level as part of the bootstrapping process. To facilitate local KDC, this automation is inserted into the Cloudera Manager CloudInit boot script. There is also a dependency for krb5.conf on the cluster hosts, prior to enabling Cloudera Manager management of these Kerberos client files. KDC setup depends on a few parameters which can be modified prior to deployment: 13 | 14 | * boot.sh 15 | * kdc_server - This is the hostname where KDC is deployed (defaults to Cloudera Manager host) 16 | * realm - This is set to hadoop.com by default. 17 | * REALM - This is set to HADOOP.COM by default. 18 | * cms_mysql.sh 19 | * KERBEROS_PASSWORD - This is used for the root/admin account. 20 | * SCM_USER_PASSWORD - By default the cloudera-scm user is given admin control of the KDC. This is required for Cloudera Manager to setup and manage principals, and the password here is used by that account. 21 | * kdc_server - Defaults to local hostname. 22 | * realm - This is set to hadoop.com by default. 23 | * REALM - This is set to HADOOP.COM by default. 24 | * cms_postgres.sh - Same items as cm_boot_mysql.sh 25 | * deploy_on_oci.py 26 | * realm - This is HADOOP.COM by default. 27 | * kdc_admin - Set to cloudera-scm@HADOOP.COM by default. 28 | * kdc_password - This should match what is set in the CM boot script for SCM_USER_PASSWORD. 29 | 30 | It is highly suggested you modify at a minimum the default passwords prior to deployment. 31 | 32 | ## CAUTION WHEN MODIFYING BOOT SCRIPTS 33 | Because boot.sh and cms_mysql.sh/cms_postgres.sh are invoked as part of user_data and extended_metadata in Terraform, if you modify these files and re-run a deployment, default behavior is existing instances will be destroyed and re-deployed because of this change. 34 | -------------------------------------------------------------------------------- /scripts/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | LOG_FILE="/var/log/cloudera-OCI-initialize.log" 3 | log() { 4 | echo "$(date) [${EXECNAME}]: $*" >> "${LOG_FILE}" 5 | } 6 | cm_fqdn=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cloudera_manager` 7 | fqdn_fields=`echo -e $cm_fqdn | gawk -F '.' '{print NF}'` 8 | cluster_domain=`echo -e $cm_fqdn | cut -d '.' -f 3-${fqdn_fields}` 9 | cloudera_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cloudera_version` 10 | cloudera_major_version=`echo $cloudera_version | cut -d '.' -f1` 11 | cm_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_version` 12 | cm_major_version=`echo $cm_version | cut -d '.' -f1` 13 | block_volume_count=`curl -L http://169.254.169.254/opc/v1/instance/metadata/block_volume_count` 14 | objectstoreRAID=`curl -L http://169.254.169.254/opc/v1/instance/metadata/objectstoreRAID` 15 | if [ $objectstoreRAID = "true" ]; then 16 | block_volume_count=$((block_volume_count+4)) 17 | fi 18 | enable_secondary_vnic=`curl -L http://169.254.169.254/opc/v1/instance/metadata/enable_secondary_vnic` 19 | if [ $enable_secondary_vnic = "true" ]; then 20 | EXECNAME="SECONDARY VNIC" 21 | host_shape=` curl -L http://169.254.169.254/opc/v1/instance/shape` 22 | case ${host_shape} in 23 | BM.HPC2.36) 24 | log "-> Skipping setup, RDMA setup not implemented" 25 | ;; 26 | 27 | *) 28 | log "->Download setup script" 29 | wget https://docs.cloud.oracle.com/en-us/iaas/Content/Resources/Assets/secondary_vnic_all_configure.sh 30 | mkdir -p /opt/oci/ 31 | mv secondary_vnic_all_configure.sh /opt/oci/ 32 | chmod +x /opt/oci/secondary_vnic_all_configure.sh 33 | log "->Configure" 34 | /opt/oci/secondary_vnic_all_configure.sh -c >> $LOG_FILE 35 | log "->rc.local enable" 36 | echo "/opt/oci/secondary_vnic_all_configure.sh -c" >> /etc/rc.local 37 | ;; 38 | esac 39 | fi 40 | EXECNAME="TUNING" 41 | log "->TUNING START" 42 | sed -i.bak 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config 43 | setenforce 0 44 | EXECNAME="PAUSE FOR YUM" 45 | log "->Waiting 120 seconds to ensure YUM is ready to go" 46 | sleep 120 47 | EXECNAME="JAVA" 48 | log "->INSTALL" 49 | yum install java-1.8.0-openjdk.x86_64 -y >> $LOG_FILE 50 | EXECNAME="NSCD" 51 | log "->INSTALL" 52 | yum install nscd -y >> $LOG_FILE 53 | systemctl start nscd.service 54 | EXECNAME="KERBEROS" 55 | log "->INSTALL" 56 | yum install krb5-workstation -y >> $LOG_FILE 57 | log "->krb5.conf" 58 | kdc_fqdn=${cm_fqdn} 59 | realm="hadoop.com" 60 | REALM="HADOOP.COM" 61 | log "-> CONFIG" 62 | rm -f /etc/krb5.conf 63 | cat > /etc/krb5.conf << EOF 64 | # Configuration snippets may be placed in this directory as well 65 | includedir /etc/krb5.conf.d/ 66 | 67 | [libdefaults] 68 | default_realm = ${REALM} 69 | dns_lookup_realm = false 70 | dns_lookup_kdc = false 71 | rdns = false 72 | ticket_lifetime = 24h 73 | renew_lifetime = 7d 74 | forwardable = true 75 | udp_preference_limit = 1000000 76 | default_tkt_enctypes = rc4-hmac 77 | default_tgs_enctypes = rc4-hmac 78 | permitted_enctypes = rc4-hmac 79 | 80 | [realms] 81 | ${REALM} = { 82 | kdc = ${kdc_fqdn}:88 83 | admin_server = ${kdc_fqdn}:749 84 | default_domain = ${realm} 85 | } 86 | 87 | [domain_realm] 88 | .${realm} = ${REALM} 89 | ${realm} = ${REALM} 90 | bastion1.${cluster_domain} = ${REALM} 91 | .bastion1.${cluster_domain} = ${REALM} 92 | bastion2.${cluster_domain} = ${REALM} 93 | .bastion2.${cluster_domain} = ${REALM} 94 | bastion3.${cluster_domain} = ${REALM} 95 | .bastion3.${cluster_domain} = ${REALM} 96 | .public1.${cluster_domain} = ${REALM} 97 | public1.${cluster_domain} = ${REALM} 98 | .public2.${cluster_domain} = ${REALM} 99 | public2.${cluster_domain} = ${REALM} 100 | .public3.${cluster_domain} = ${REALM} 101 | public3.${cluster_domain} = ${REALM} 102 | .private1.${cluster_domain} = ${REALM} 103 | private1.${cluster_domain} = ${REALM} 104 | .private2.${cluster_domain} = ${REALM} 105 | private2.${cluster_domain} = ${REALM} 106 | .private3.${cluster_domain} = ${REALM} 107 | private3.${cluster_domain} = ${REALM} 108 | 109 | [kdc] 110 | profile = /var/kerberos/krb5kdc/kdc.conf 111 | 112 | [logging] 113 | kdc = FILE:/var/log/krb5kdc.log 114 | admin_server = FILE:/var/log/kadmin.log 115 | default = FILE:/var/log/krb5lib.log 116 | EOF 117 | EXECNAME="TUNING" 118 | log "->OS" 119 | echo never | tee -a /sys/kernel/mm/transparent_hugepage/enabled 120 | echo "echo never | tee -a /sys/kernel/mm/transparent_hugepage/enabled" | tee -a /etc/rc.local 121 | echo vm.swappiness=1 | tee -a /etc/sysctl.conf 122 | echo 1 | tee /proc/sys/vm/swappiness 123 | echo net.ipv4.tcp_timestamps=0 >> /etc/sysctl.conf 124 | echo net.ipv4.tcp_sack=1 >> /etc/sysctl.conf 125 | echo net.core.rmem_max=4194304 >> /etc/sysctl.conf 126 | echo net.core.wmem_max=4194304 >> /etc/sysctl.conf 127 | echo net.core.rmem_default=4194304 >> /etc/sysctl.conf 128 | echo net.core.wmem_default=4194304 >> /etc/sysctl.conf 129 | echo net.core.optmem_max=4194304 >> /etc/sysctl.conf 130 | echo net.ipv4.tcp_rmem="4096 87380 4194304" >> /etc/sysctl.conf 131 | echo net.ipv4.tcp_wmem="4096 65536 4194304" >> /etc/sysctl.conf 132 | echo net.ipv4.tcp_low_latency=1 >> /etc/sysctl.conf 133 | sed -i "s/defaults 1 1/defaults,noatime 0 0/" /etc/fstab 134 | echo "hdfs - nofile 32768 135 | hdfs - nproc 2048 136 | hbase - nofile 32768 137 | hbase - nproc 2048" >> /etc/security/limits.conf 138 | ulimit -n 262144 139 | log "->FirewallD" 140 | systemctl stop firewalld 141 | systemctl disable firewalld 142 | EXECNAME="MYSQL Connector" 143 | log "->INSTALL" 144 | wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz 145 | tar zxvf mysql-connector-java-5.1.46.tar.gz 146 | mkdir -p /usr/share/java/ 147 | cd mysql-connector-java-5.1.46 148 | cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar 149 | # Disk Setup Functions 150 | vol_match() { 151 | case $i in 152 | 1) disk="oraclevdb";; 153 | 2) disk="oraclevdc";; 154 | 3) disk="oraclevdd";; 155 | 4) disk="oraclevde";; 156 | 5) disk="oraclevdf";; 157 | 6) disk="oraclevdg";; 158 | 7) disk="oraclevdh";; 159 | 8) disk="oraclevdi";; 160 | 9) disk="oraclevdj";; 161 | 10) disk="oraclevdk";; 162 | 11) disk="oraclevdl";; 163 | 12) disk="oraclevdm";; 164 | 13) disk="oraclevdn";; 165 | 14) disk="oraclevdo";; 166 | 15) disk="oraclevdp";; 167 | 16) disk="oraclevdq";; 168 | 17) disk="oraclevdr";; 169 | 18) disk="oraclevds";; 170 | 19) disk="oraclevdt";; 171 | 20) disk="oraclevdu";; 172 | 21) disk="oraclevdv";; 173 | 22) disk="oraclevdw";; 174 | 23) disk="oraclevdx";; 175 | 24) disk="oraclevdy";; 176 | 25) disk="oraclevdz";; 177 | 26) disk="oraclevdab";; 178 | 27) disk="oraclevdac";; 179 | 28) disk="oraclevdad";; 180 | 29) disk="oraclevdae";; 181 | 30) disk="oraclevdaf";; 182 | 31) disk="oraclevdag";; 183 | esac 184 | } 185 | iscsi_detection() { 186 | iscsiadm -m discoverydb -D -t sendtargets -p 169.254.2.$i:3260 2>&1 2>/dev/null 187 | iscsi_chk=`echo -e $?` 188 | if [ $iscsi_chk = "0" ]; then 189 | iqn[${i}]=`iscsiadm -m discoverydb -D -t sendtargets -p 169.254.2.${i}:3260 | gawk '{print $2}'` 190 | log "-> Discovered volume $((i-1)) - IQN: ${iqn[${i}]}" 191 | continue 192 | else 193 | volume_count="${#iqn[@]}" 194 | log "--> Discovery Complete - ${#iqn[@]} volumes found" 195 | fi 196 | } 197 | iscsi_setup() { 198 | log "-> ISCSI Volume Setup - Volume ${i} : IQN ${iqn[$n]}" 199 | iscsiadm -m node -o new -T ${iqn[$n]} -p 169.254.2.${n}:3260 200 | log "--> Volume ${iqn[$n]} added" 201 | iscsiadm -m node -o update -T ${iqn[$n]} -n node.startup -v automatic 202 | log "--> Volume ${iqn[$n]} startup set" 203 | iscsiadm -m node -T ${iqn[$n]} -p 169.254.2.${n}:3260 -l 204 | log "--> Volume ${iqn[$n]} done" 205 | } 206 | EXECNAME="DISK DETECTION" 207 | log "->Begin Block Volume Detection Loop" 208 | detection_flag="0" 209 | while [ "$detection_flag" = "0" ]; do 210 | log "-- Detecting Block Volumes --" 211 | for i in `seq 2 33`; do 212 | if [ -z $volume_count ]; then 213 | iscsi_detection 214 | fi 215 | done; 216 | master_check=`hostname | grep master && echo -e $?` 217 | bastion_check=`hostname | grep bastion && echo -e $?` 218 | if [ $master_check = "0" ]; then 219 | total_volume_count=3 220 | elif [ $bastion_check = "0" ]; then 221 | total_volume_count=2 222 | else 223 | total_volume_count=$((block_volume_count+2)) 224 | fi 225 | log "-- $total_volume_count volumes expected $volume_count volumes found --" 226 | if [ "$volume_count" = "0" ]; then 227 | log "-- $volume_count Block Volumes detected, sleeping 15 then retry --" 228 | unset volume_count 229 | unset iqn 230 | sleep 15 231 | continue 232 | elif [ "$volume_count" != "$total_volume_count" ]; then 233 | log "-- Sanity Check Failed - $volume_count Volumes found, $total_volume_count expected. Re-running --" 234 | unset volume_count 235 | unset iqn 236 | sleep 15 237 | continue 238 | else 239 | log "-- Setup for ${#iqn[@]} Block Volumes --" 240 | for i in `seq 1 ${#iqn[@]}`; do 241 | n=$((i+1)) 242 | iscsi_setup 243 | done; 244 | detection_flag="1" 245 | fi 246 | done; 247 | 248 | EXECNAME="DISK PROVISIONING" 249 | data_mount () { 250 | log "-->Mounting /dev/$disk to /data$dcount" 251 | mkdir -p /data$dcount 252 | mount -o noatime,barrier=1 -t ext4 /dev/$disk /data$dcount 253 | UUID=`blkid /dev/$disk | cut -d '"' -f2` 254 | echo "UUID=$UUID /data$dcount ext4 defaults,noatime,discard,barrier=0 0 1" | tee -a /etc/fstab 255 | } 256 | 257 | block_data_mount () { 258 | log "-->Mounting /dev/oracleoci/$disk to /data$dcount" 259 | mkdir -p /data$dcount 260 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /data$dcount 261 | UUID=`blkid /dev/oracleoci/$disk | cut -d '"' -f 2` 262 | if [ ! -z $UUID ]; then 263 | echo "UUID=$UUID /data$dcount ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 264 | fi 265 | } 266 | raid_disk_setup() { 267 | sed -e 's/\s*\([\+0-9a-zA-Z]*\).*/\1/' << EOF | fdisk /dev/oracleoci/$disk 268 | n 269 | p 270 | 1 271 | 272 | 273 | t 274 | fd 275 | w 276 | EOF 277 | } 278 | EXECNAME="DISK SETUP" 279 | log "->Checking for disks..." 280 | dcount=0 281 | for disk in `ls /dev/ | grep nvme | grep n1`; do 282 | log "-->Processing /dev/$disk" 283 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/$disk 284 | data_mount 285 | dcount=$((dcount+1)) 286 | done; 287 | if [ ${#iqn[@]} -gt 0 ]; then 288 | for i in `seq 1 ${#iqn[@]}`; do 289 | n=$((i+1)) 290 | dsetup="0" 291 | while [ $dsetup = "0" ]; do 292 | vol_match 293 | log "-->Checking /dev/oracleoci/$disk" 294 | if [ -h /dev/oracleoci/$disk ]; then 295 | case $disk in 296 | oraclevdb) 297 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk 298 | log "--->Mounting /dev/oracleoci/$disk to /var/log/cloudera" 299 | mkdir -p /var/log/cloudera 300 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /var/log/cloudera 301 | echo "/dev/oracleoci/oraclevdb /var/log/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 302 | mkdir -p /var/log/cloudera/cloudera-scm-agent 303 | ln -s /var/log/cloudera/cloudera-scm-agent /var/log/cloudera-scm-agent 304 | ;; 305 | oraclevdc) 306 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk 307 | log "--->Mounting /dev/oracleoci/$disk to /opt/cloudera" 308 | mkdir -p /opt/cloudera 309 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /opt/cloudera 310 | echo "/dev/oracleoci/oraclevdc /opt/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 311 | ;; 312 | oraclevdd|oraclevde|oraclevdf|oraclevdg) 313 | if [ $objectstoreRAID = "true" ]; then 314 | raid_disk_setup 315 | else 316 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk 317 | block_data_mount 318 | dcount=$((dcount+1)) 319 | fi 320 | ;; 321 | *) 322 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk 323 | block_data_mount 324 | dcount=$((dcount+1)) 325 | ;; 326 | esac 327 | /sbin/tune2fs -i0 -c0 /dev/oracleoci/$disk 328 | unset UUID 329 | dsetup="1" 330 | else 331 | log "--->${disk} not found, running ISCSI again." 332 | log "-- Re-Running Detection & Setup Block Volumes --" 333 | detection_done="0" 334 | log "-- Detecting Block Volumes --" 335 | for i in `seq 2 33`; do 336 | if [ $detection_done = "0" ]; then 337 | iscsi_detection 338 | fi 339 | done; 340 | for j in `seq 1 ${#iqn[@]}`; do 341 | n=$((j+1)) 342 | iscsi_setup 343 | done 344 | fi 345 | done; 346 | done; 347 | fi 348 | if [ $objectstoreRAID = "true" ]; then 349 | EXECNAME="TMP" 350 | log "->Setup LVM" 351 | vgcreate RAID0 /dev/oracleoci/oraclevd[d-g]1 352 | lvcreate -i 2 -I 64 -l 100%FREE -n tmp RAID0 353 | mkfs.ext4 /dev/RAID0/tmp 354 | mkdir -p /mnt/tmp 355 | chmod 1777 /mnt/tmp 356 | mount /dev/RAID0/tmp /mnt/tmp 357 | mount -B /tmp /mnt/tmp 358 | chmod 1777 /tmp 359 | echo "/dev/RAID0/tmp /tmp ext4 defaults,_netdev,noatime,discard,barrier=0 0 0" | tee -a /etc/fstab 360 | fi 361 | EXECNAME="Cloudera Agent Install" 362 | if [ ${cm_major_version} = "7" ]; then 363 | log "-->CDP install detected - CM $cm_version" 364 | rpm --import https://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/RPM-GPG-KEY-cloudera 365 | wget https://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/cloudera-manager-trial.repo -O /etc/yum.repos.d/cloudera-manager.repo 366 | else 367 | log "-->Setup GPG Key & CM ${cm_version} repo" 368 | rpm --import https://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/RPM-GPG-KEY-cloudera 369 | wget http://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/cloudera-manager.repo -O /etc/yum.repos.d/cloudera-manager.repo 370 | fi 371 | yum install cloudera-manager-agent -y >> $LOG_FILE 372 | export JDK=`ls /usr/lib/jvm | head -n 1` 373 | sudo JAVA_HOME=/usr/lib/jvm/$JDK/jre/ /opt/cloudera/cm-agent/bin/certmanager setup --configure-services 374 | cp /etc/cloudera-scm-agent/config.ini /etc/cloudera-scm-agent/config.ini.orig 375 | sed -e "s/\(server_host=\).*/\1${cm_fqdn}/" -i /etc/cloudera-scm-agent/config.ini 376 | if [ ${enable_secondary_vnic} = "true" ]; then 377 | agent_hostname=`curl -L http://169.254.169.254/opc/v1/instance/metadata/agent_hostname` 378 | sed -i 's,# listening_hostname=,'"listening_hostname=${agent_hostname}"',g' /etc/cloudera-scm-agent/config.ini 379 | agent_ip=`host ${agent_hostname} | gawk '{print $4}'` 380 | sed -i 's,# listening_ip=,'"listening_ip=${agent_ip}"',g' /etc/cloudera-scm-agent/config.ini 381 | fi 382 | systemctl start cloudera-scm-agent 383 | EXECNAME="END" 384 | log "->DONE" 385 | -------------------------------------------------------------------------------- /scripts/cloudera_manager_boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | LOG_FILE="/var/log/cloudera-OCI-initialize.log" 3 | log() { 4 | echo "$(date) [${EXECNAME}]: $*" >> "${LOG_FILE}" 5 | } 6 | EXECNAME="Metadata Extraction" 7 | log "->Deployment Script Decode" 8 | curl -L http://169.254.169.254/opc/v1/instance/metadata/deploy_on_oci | base64 -d > /var/lib/cloud/instance/scripts/deploy_on_oci.py.gz 9 | log "-->Extract" 10 | gunzip /var/lib/cloud/instance/scripts/deploy_on_oci.py.gz >> $LOG_FILE 11 | log "->CMS Setup Script Decode" 12 | curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_install | base64 -d > /var/lib/cloud/instance/scripts/cms_mysql.sh.gz 13 | log"-->Extract" 14 | gunzip /var/lib/cloud/instance/scripts/cms_mysql.sh.gz 15 | chmod +x /var/lib/cloud/instance/scripts/cms_mysql.sh 16 | EXECNAME="CMS Setup" 17 | log "->Execute" 18 | cd /var/lib/cloud/instance/scripts/ 19 | ./cms_mysql.sh 20 | -------------------------------------------------------------------------------- /scripts/cms_mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LOG_FILE="/var/log/cloudera-OCI-initialize.log" 4 | log() { 5 | echo "$(date) [${EXECNAME}]: $*" >> "${LOG_FILE}" 6 | } 7 | cm_fqdn=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cloudera_manager` 8 | fqdn_fields=`echo -e $cm_fqdn | gawk -F '.' '{print NF}'` 9 | cluster_domain=`echo -e $cm_fqdn | cut -d '.' -f 3-${fqdn_fields}` 10 | cm_ip=`host ${cm_fqdn} | gawk '{print $4}'` 11 | cluster_subnet=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cluster_subnet` 12 | bastion_subnet=`curl -L http://169.254.169.254/opc/v1/instance/metadata/bastion_subnet` 13 | utility_subnet=`curl -L http://169.254.169.254/opc/v1/instance/metadata/utility_subnet` 14 | cloudera_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cloudera_version` 15 | cloudera_major_version=`echo $cloudera_version | cut -d '.' -f1` 16 | cm_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_version` 17 | cm_major_version=`echo $cm_version | cut -d '.' -f1` 18 | # Note that the AD detection depends on the subnet containing the AD as the last character in the name 19 | worker_shape=`curl -L http://169.254.169.254/opc/v1/instance/metadata/worker_shape` 20 | worker_disk_count=`curl -L http://169.254.169.254/opc/v1/instance/metadata/block_volume_count` 21 | secure_cluster=`curl -L http://169.254.169.254/opc/v1/instance/metadata/secure_cluster` 22 | hdfs_ha=`curl -L http://169.254.169.254/opc/v1/instance/metadata/hdfs_ha` 23 | cluster_name=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cluster_name` 24 | cm_username=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_username` 25 | cm_password=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_password` 26 | vcore_ratio=`curl -L http://169.254.169.254/opc/v1/instance/metadata/vcore_ratio` 27 | debug=`curl -L http://169.254.169.254/opc/v1/instance/metadata/enable_debug` 28 | yarn_scheduler=`curl -L http://169.254.169.254/opc/v1/instance/metadata/yarn_scheduler` 29 | full_service_list=(ATLAS HBASE HDFS HIVE IMPALA KAFKA OOZIE RANGER SOLR SPARK_ON_YARN SQOOP_CLIENT YARN) 30 | service_list="ZOOKEEPER" 31 | rangeradmin_password='' 32 | for service in ${full_service_list[@]}; do 33 | svc_check=`curl -L http://169.254.169.254/opc/v1/instance/metadata/svc_${service}` 34 | if [ $svc_check = "true" ]; then 35 | if [ $service = "RANGER" ]; then 36 | rangeradmin_password=`curl -L http://169.254.169.254/opc/v1/instance/metadata/rangeradmin_password` 37 | ranger_enabled="true" 38 | service_list=`echo -e "${service_list},${service}"` 39 | elif [ $service = "ATLAS" ]; then 40 | atlas_enabled="true" 41 | service_list=`echo -e "${service_list},${service}"` 42 | else 43 | service_list=`echo -e "${service_list},${service}"` 44 | fi 45 | 46 | fi 47 | done; 48 | EXECNAME="TUNING" 49 | log "-> START" 50 | sed -i.bak 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config 51 | setenforce 0 52 | echo never | tee -a /sys/kernel/mm/transparent_hugepage/enabled 53 | echo "echo never | tee -a /sys/kernel/mm/transparent_hugepage/enabled" | tee -a /etc/rc.local 54 | echo vm.swappiness=1 | tee -a /etc/sysctl.conf 55 | echo 1 | tee /proc/sys/vm/swappiness 56 | echo net.ipv4.tcp_timestamps=0 >> /etc/sysctl.conf 57 | echo net.ipv4.tcp_sack=1 >> /etc/sysctl.conf 58 | echo net.core.rmem_max=4194304 >> /etc/sysctl.conf 59 | echo net.core.wmem_max=4194304 >> /etc/sysctl.conf 60 | echo net.core.rmem_default=4194304 >> /etc/sysctl.conf 61 | echo net.core.wmem_default=4194304 >> /etc/sysctl.conf 62 | echo net.core.optmem_max=4194304 >> /etc/sysctl.conf 63 | echo net.ipv4.tcp_rmem="4096 87380 4194304" >> /etc/sysctl.conf 64 | echo net.ipv4.tcp_wmem="4096 65536 4194304" >> /etc/sysctl.conf 65 | echo net.ipv4.tcp_low_latency=1 >> /etc/sysctl.conf 66 | sed -i "s/defaults 1 1/defaults,noatime 0 0/" /etc/fstab 67 | echo "hdfs - nofile 32768 68 | hdfs - nproc 2048 69 | hbase - nofile 32768 70 | hbase - nproc 2048" >> /etc/security/limits.conf 71 | ulimit -n 262144 72 | systemctl stop firewalld 73 | systemctl disable firewalld 74 | EXECNAME="KERBEROS" 75 | log "-> INSTALL" 76 | yum -y install krb5-server krb5-libs krb5-workstation >> $LOG_FILE 77 | KERBEROS_PASSWORD="SOMEPASSWORD" 78 | SCM_USER_PASSWORD="somepassword" 79 | kdc_fqdn=${cm_fqdn} 80 | realm="hadoop.com" 81 | REALM="HADOOP.COM" 82 | log "-> CONFIG" 83 | rm -f /etc/krb5.conf 84 | cat > /etc/krb5.conf << EOF 85 | # Configuration snippets may be placed in this directory as well 86 | includedir /etc/krb5.conf.d/ 87 | 88 | [libdefaults] 89 | default_realm = ${REALM} 90 | dns_lookup_realm = false 91 | dns_lookup_kdc = false 92 | rdns = false 93 | ticket_lifetime = 24h 94 | renew_lifetime = 7d 95 | forwardable = true 96 | udp_preference_limit = 1000000 97 | default_tkt_enctypes = rc4-hmac 98 | default_tgs_enctypes = rc4-hmac 99 | permitted_enctypes = rc4-hmac 100 | 101 | [realms] 102 | ${REALM} = { 103 | kdc = ${kdc_fqdn}:88 104 | admin_server = ${kdc_fqdn}:749 105 | default_domain = ${realm} 106 | } 107 | 108 | [domain_realm] 109 | .${realm} = ${REALM} 110 | ${realm} = ${REALM} 111 | bastion1.${cluster_domain} = ${REALM} 112 | .bastion1.${cluster_domain} = ${REALM} 113 | bastion2.${cluster_domain} = ${REALM} 114 | .bastion2.${cluster_domain} = ${REALM} 115 | bastion3.${cluster_domain} = ${REALM} 116 | .bastion3.${cluster_domain} = ${REALM} 117 | .public1.${cluster_domain} = ${REALM} 118 | public1.${cluster_domain} = ${REALM} 119 | .public2.${cluster_domain} = ${REALM} 120 | public2.${cluster_domain} = ${REALM} 121 | .public3.${cluster_domain} = ${REALM} 122 | public3.${cluster_domain} = ${REALM} 123 | .private1.${cluster_domain} = ${REALM} 124 | private1.${cluster_domain} = ${REALM} 125 | .private2.${cluster_domain} = ${REALM} 126 | private2.${cluster_domain} = ${REALM} 127 | .private3.${cluster_domain} = ${REALM} 128 | private3.${cluster_domain} = ${REALM} 129 | 130 | [kdc] 131 | profile = /var/kerberos/krb5kdc/kdc.conf 132 | 133 | [logging] 134 | kdc = FILE:/var/log/krb5kdc.log 135 | admin_server = FILE:/var/log/kadmin.log 136 | default = FILE:/var/log/krb5lib.log 137 | EOF 138 | rm -f /var/kerberos/krb5kdc/kdc.conf 139 | cat > /var/kerberos/krb5kdc/kdc.conf << EOF 140 | default_realm = ${REALM} 141 | 142 | [kdcdefaults] 143 | v4_mode = nopreauth 144 | kdc_ports = 0 145 | 146 | [realms] 147 | ${REALM} = { 148 | kdc_ports = 88 149 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 150 | database_name = /var/kerberos/krb5kdc/principal 151 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 152 | key_stash_file = /var/kerberos/krb5kdc/stash 153 | max_life = 10h 0m 0s 154 | max_renewable_life = 7d 0h 0m 0s 155 | master_key_type = des3-hmac-sha1 156 | supported_enctypes = rc4-hmac:normal 157 | default_principal_flags = +preauth 158 | } 159 | EOF 160 | rm -f /var/kerberos/krb5kdc/kadm5.acl 161 | cat > /var/kerberos/krb5kdc/kadm5.acl << EOF 162 | */admin@${REALM} * 163 | cloudera-scm@${REALM} * 164 | EOF 165 | kdb5_util create -r ${REALM} -s -P ${KERBEROS_PASSWORD} >> $LOG_FILE 166 | echo -e "addprinc root/admin\n${KERBEROS_PASSWORD}\n${KERBEROS_PASSWORD}\naddprinc cloudera-scm\n${SCM_USER_PASSWORD}\n${SCM_USER_PASSWORD}\nktadd -k /var/kerberos/krb5kdc/kadm5.keytab kadmin/admin\nktadd -k /var/kerberos/krb5kdc/kadm5.keytab kadmin/changepw\nexit\n" | kadmin.local -r ${REALM} 167 | log "-> START" 168 | systemctl start krb5kdc.service >> $LOG_FILE 169 | systemctl start kadmin.service >> $LOG_FILE 170 | systemctl enable krb5kdc.service >> $LOG_FILE 171 | systemctl enable kadmin.service >> $LOG_FILE 172 | 173 | EXECNAME="Cloudera Manager & Pre-Reqs Install" 174 | log "-> Installation" 175 | if [ ${cm_major_version} = "7" ]; then 176 | log "-->CDP install detected - CM version $cm_version" 177 | rpm --import https://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/RPM-GPG-KEY-cloudera 178 | wget https://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/cloudera-manager-trial.repo -O /etc/yum.repos.d/cloudera-manager.repo 179 | else 180 | log "-->Setup GPG Key & CM ${cm_version} repo" 181 | rpm --import https://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/RPM-GPG-KEY-cloudera 182 | wget http://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/cloudera-manager.repo -O /etc/yum.repos.d/cloudera-manager.repo 183 | fi 184 | yum install cloudera-manager-server java-1.8.0-openjdk.x86_64 python-pip nscd -y >> $LOG_FILE 185 | pip install psycopg2==2.7.5 --ignore-installed >> $LOG_FILE 186 | yum install oracle-j2sdk1.8.x86_64 cloudera-manager-daemons cloudera-manager-agent -y >> $LOG_FILE 187 | cp /etc/cloudera-scm-agent/config.ini /etc/cloudera-scm-agent/config.ini.orig 188 | sed -e "s/\(server_host=\).*/\1${cm_fqdn}/" -i /etc/cloudera-scm-agent/config.ini 189 | #export JDK=`ls /usr/lib/jvm | head -n 1` 190 | #sudo JAVA_HOME=/usr/lib/jvm/$JDK/jre/ /opt/cloudera/cm-agent/bin/certmanager setup --configure-services 191 | chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-agent/ 192 | systemctl start nscd.service 193 | systemctl start cloudera-scm-agent 194 | 195 | create_random_password() 196 | { 197 | perl -le 'print map { ("a".."z", "A".."Z", 0..9)[rand 62] } 1..10' 198 | } 199 | EXECNAME="MySQL DB" 200 | log "->Install" 201 | wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm 202 | rpm -ivh mysql-community-release-el7-5.noarch.rpm 203 | yum install mysql-server -y 204 | log "->Tuning" 205 | wget https://raw.githubusercontent.com/oracle-quickstart/oci-cloudera/master/scripts/my.cnf 206 | mv my.cnf /etc/my.cnf 207 | log "->Start" 208 | systemctl enable mysqld 209 | systemctl start mysqld 210 | log "->Bootstrap Databases" 211 | mysql_pw=` cat /var/log/mysqld.log | grep root@localhost | gawk '{print $13}'` 212 | echo -e "$mysql_pw" >> /etc/mysql/mysql_root.pw 213 | mysql -u root --connect-expired-password -p${mysql_pw} -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'S0m3p@ssw1234';" 214 | mysql -u root --connect-expired-password -p${mysql_pw} -e "FLUSH PRIVILEGES;" 215 | mysql_pw="S0m3p@ssw1234" 216 | mysql -u root -p${mysql_pw} -e "SET GLOBAL validate_password.policy=LOW;" 217 | mysql -u root -p${mysql_pw} -e "SET GLOBAL log_bin_trust_function_creators = 1;" 218 | mkdir -p /etc/mysql 219 | for DATABASE in "scm" "amon" "rman" "hue" "metastore" "sentry" "nav" "navms" "oozie" "ranger" "atlas"; do 220 | pw=$(create_random_password) 221 | if [ ${DATABASE} = "metastore" ]; then 222 | USER="hive" 223 | elif [ ${DATABASE} = "ranger" ]; then 224 | if [ ${ranger_enabled} = "true" ]; then 225 | USER="rangeradmin" 226 | else 227 | continue; 228 | fi 229 | elif [ ${DATABASE} = "atlas" ]; then 230 | if [ ${atlas_enabled} = "true" ]; then 231 | USER="atlas" 232 | else 233 | continue 234 | fi 235 | else 236 | USER=${DATABASE} 237 | fi 238 | echo -e "CREATE DATABASE ${DATABASE} DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" >> /etc/mysql/cloudera.sql 239 | echo -e "CREATE USER \'${USER}\'@'%' IDENTIFIED BY \'${pw}\';" >> /etc/mysql/cloudera.sql 240 | echo -e "GRANT ALL on ${DATABASE}.* to \'${USER}\'@'%';" >> /etc/mysql/cloudera.sql 241 | echo "${USER}:${pw}" >> /etc/mysql/mysql.pw 242 | done; 243 | sed -i 's/\\//g' /etc/mysql/cloudera.sql 244 | mysql -u root -p${mysql_pw} < /etc/mysql/cloudera.sql 245 | mysql -u root -p${mysql_pw} -e "FLUSH PRIVILEGES" 246 | log "->Java Connector" 247 | wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz 248 | tar zxvf mysql-connector-java-5.1.46.tar.gz 249 | mkdir -p /usr/share/java/ 250 | cd mysql-connector-java-5.1.46 251 | cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar 252 | log "->SCM Prepare DB" 253 | for user in `cat /etc/mysql/mysql.pw | gawk -F ':' '{print $1}'`; do 254 | log "-->${user} preparation" 255 | pw=`cat /etc/mysql/mysql.pw | grep -w $user | cut -d ':' -f 2` 256 | if [ $user = "hive" ]; then 257 | database="metastore" 258 | elif [ $user = "rangeradmin" ]; then 259 | database="ranger" 260 | else 261 | database=${user} 262 | fi 263 | /opt/cloudera/cm/schema/scm_prepare_database.sh mysql ${database} ${user} ${pw} 264 | done; 265 | vol_match() { 266 | case $i in 267 | 1) disk="oraclevdb";; 268 | 2) disk="oraclevdc";; 269 | 3) disk="oraclevdd";; 270 | 4) disk="oraclevde";; 271 | 5) disk="oraclevdf";; 272 | 6) disk="oraclevdg";; 273 | 7) disk="oraclevdh";; 274 | 8) disk="oraclevdi";; 275 | 9) disk="oraclevdj";; 276 | 10) disk="oraclevdk";; 277 | 11) disk="oraclevdl";; 278 | 12) disk="oraclevdm";; 279 | 13) disk="oraclevdn";; 280 | 14) disk="oraclevdo";; 281 | 15) disk="oraclevdp";; 282 | 16) disk="oraclevdq";; 283 | 17) disk="oraclevdr";; 284 | 18) disk="oraclevds";; 285 | 19) disk="oraclevdt";; 286 | 20) disk="oraclevdu";; 287 | 21) disk="oraclevdv";; 288 | 22) disk="oraclevdw";; 289 | 23) disk="oraclevdx";; 290 | 24) disk="oraclevdy";; 291 | 25) disk="oraclevdz";; 292 | 26) disk="oraclevdab";; 293 | 27) disk="oraclevdac";; 294 | 28) disk="oraclevdad";; 295 | 29) disk="oraclevdae";; 296 | 30) disk="oraclevdaf";; 297 | 31) disk="oraclevdag";; 298 | esac 299 | } 300 | iscsi_setup() { 301 | log "-> ISCSI Volume Setup - Volume $i : IQN ${iqn[$n]}" 302 | iscsiadm -m node -o new -T ${iqn[$n]} -p 169.254.2.${n}:3260 303 | log "--> Volume ${iqn[$n]} added" 304 | iscsiadm -m node -o update -T ${iqn[$n]} -n node.startup -v automatic 305 | log "--> Volume ${iqn[$n]} startup set" 306 | iscsiadm -m node -T ${iqn[$n]} -p 169.254.2.${n}:3260 -l 307 | log "--> Volume ${iqn[$n]} done" 308 | } 309 | iscsi_target_only(){ 310 | log "-->Logging into Volume ${iqn[$n]}" 311 | iscsiadm -m node -T ${iqn[$n]} -p 169.254.2.${n}:3260 -l 312 | } 313 | EXECNAME="ISCSI" 314 | done="0" 315 | log "-- Detecting Block Volumes --" 316 | for i in `seq 2 33`; do 317 | if [ $done = "0" ]; then 318 | iscsiadm -m discoverydb -D -t sendtargets -p 169.254.2.${i}:3260 2>&1 2>/dev/null 319 | iscsi_chk=`echo -e $?` 320 | if [ $iscsi_chk = "0" ]; then 321 | iqn[$i]=`iscsiadm -m discoverydb -D -t sendtargets -p 169.254.2.${i}:3260 | gawk '{print $2}'` 322 | log "-> Discovered volume $((i-1)) - IQN: ${iqn[$i]}" 323 | continue 324 | else 325 | log "--> Discovery Complete - ${#iqn[@]} volumes found" 326 | done="1" 327 | fi 328 | fi 329 | done; 330 | log "-- Setup for ${#iqn[@]} Block Volumes --" 331 | if [ ${#iqn[@]} -gt 0 ]; then 332 | for i in `seq 1 ${#iqn[@]}`; do 333 | n=$((i+1)) 334 | iscsi_setup 335 | done; 336 | fi 337 | EXECNAME="DISK PROVISIONING" 338 | data_mount () { 339 | log "-->Mounting /dev/$disk to /data$dcount" 340 | mkdir -p /data$dcount 341 | mount -o noatime,barrier=1 -t ext4 /dev/$disk /data$dcount 342 | UUID=`blkid /dev/$disk | cut -d '"' -f 2` 343 | echo "UUID=$UUID /data$dcount ext4 defaults,noatime,discard,barrier=0 0 1" | tee -a /etc/fstab 344 | } 345 | block_data_mount () { 346 | log "-->Mounting /dev/oracleoci/$disk to /data$dcount" 347 | mkdir -p /data$dcount 348 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /data$dcount 349 | UUID=`blkid /dev/oracleoci/$disk | cut -d '"' -f 2` 350 | echo "UUID=$UUID /data$dcount ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 351 | } 352 | log "->Checking for disks..." 353 | dcount=0 354 | for disk in `cat /proc/partitions | grep nv`; do 355 | log "-->Processing /dev/$disk" 356 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/$disk 357 | data_mount 358 | dcount=$((dcount+1)) 359 | done; 360 | if [ ${#iqn[@]} -gt 0 ]; then 361 | for i in `seq 1 ${#iqn[@]}`; do 362 | n=$((i+1)) 363 | dsetup="0" 364 | while [ $dsetup = "0" ]; do 365 | vol_match 366 | log "-->Checking /dev/oracleoci/$disk" 367 | if [ -h /dev/oracleoci/$disk ]; then 368 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk 369 | if [ $disk = "oraclevdb" ]; then 370 | log "--->Mounting /dev/oracleoci/$disk to /var/log/cloudera" 371 | mkdir -p /var/log/cloudera 372 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /var/log/cloudera 373 | echo "/dev/oracleoci/oraclevdb /var/log/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 374 | elif [ $disk = "oraclevdc" ]; then 375 | log "--->Mounting /dev/oracleoci/$disk to /opt/cloudera" 376 | if [ -d /opt/cloudera ]; then 377 | mv /opt/cloudera /opt/cloudera_pre 378 | mkdir -p /opt/cloudera 379 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /opt/cloudera 380 | mv /opt/cloudera_pre/* /opt/cloudera 381 | rm -fR /opt/cloudera_pre 382 | else 383 | mkdir -p /opt/cloudera 384 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /opt/cloudera 385 | fi 386 | echo "/dev/oracleoci/oraclevdc /opt/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 387 | else 388 | block_data_mount 389 | dcount=$((dcount+1)) 390 | fi 391 | /sbin/tune2fs -i0 -c0 /dev/oracleoci/$disk 392 | dsetup="1" 393 | else 394 | log "--->${disk} not found, running ISCSI setup again." 395 | iscsi_target_only 396 | sleep 5 397 | fi 398 | done; 399 | done; 400 | fi 401 | EXECNAME="Cloudera Manager" 402 | log "->Starting Cloudera Manager" 403 | chown -R cloudera-scm:cloudera-scm /etc/cloudera-scm-server 404 | systemctl start cloudera-scm-server 405 | EXECNAME="Cloudera ${cloudera_version}" 406 | log "->Installing Python Pre-reqs" 407 | sudo yum install python python-pip -y >> $LOG_FILE 408 | sudo pip install --upgrade pip >> $LOG_FILE 409 | sudo pip install cm_client >> $LOG_FILE 410 | log "->Running Cluster Deployment" 411 | log "-->Host Discovery" 412 | detection_flag="0" 413 | w=1 414 | while [ $detection_flag = "0" ]; do 415 | worker_lookup=`host cloudera-worker-$w.${cluster_subnet}.${cluster_domain}` 416 | worker_check=`echo -e $?` 417 | if [ $worker_check = "0" ]; then 418 | worker_fqdn[$w]="cloudera-worker-$w.${cluster_subnet}.${cluster_domain}" 419 | w=$((w+1)) 420 | else 421 | detection_flag="1" 422 | fi 423 | done; 424 | fqdn_list="cloudera-utility-1.${utility_subnet}.${cluster_domain},cloudera-master-1.${cluster_subnet}.${cluster_domain},cloudera-master-2.${cluster_subnet}.${cluster_domain}" 425 | num_workers=${#worker_fqdn[@]} 426 | for w in `seq 1 $num_workers`; do 427 | fqdn_list=`echo "${fqdn_list},${worker_fqdn[$w]}"` 428 | done; 429 | log "-->Host List: ${fqdn_list}" 430 | log "-->Cluster Build" 431 | XOPTS='' 432 | if [ ${ranger_enabled} = "true" ]; then 433 | XOPTS="-R ${rangeradmin_password}" 434 | fi 435 | if [ ${secure_cluster} = "true" ]; then 436 | XOPTS="${XOPTS} -S" 437 | fi 438 | if [ ${hdfs_ha} = "true" ]; then 439 | XOPTS="${XOPTS} -H" 440 | fi 441 | if [ ${debug} = "true" ]; then 442 | XOPTS="${XOPTS} -D" 443 | fi 444 | log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -M mysql -Y ${yarn_scheduler} ${XOPTS}" 445 | python /var/lib/cloud/instance/scripts/deploy_on_oci.py -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -M mysql -Y ${yarn_scheduler} ${XOPTS} 2>&1 >> $LOG_FILE 446 | log "->DONE" 447 | -------------------------------------------------------------------------------- /scripts/cms_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LOG_FILE="/var/log/cloudera-OCI-initialize.log" 4 | log() { 5 | echo "$(date) [${EXECNAME}]: $*" >> "${LOG_FILE}" 6 | } 7 | cm_fqdn=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cloudera_manager` 8 | fqdn_fields=`echo -e $cm_fqdn | gawk -F '.' '{print NF}'` 9 | cluster_domain=`echo -e $cm_fqdn | cut -d '.' -f 3-${fqdn_fields}` 10 | cm_ip=`host ${cm_fqdn} | gawk '{print $4}'` 11 | cluster_subnet=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cluster_subnet` 12 | bastion_subnet=`curl -L http://169.254.169.254/opc/v1/instance/metadata/bastion_subnet` 13 | utility_subnet=`curl -L http://169.254.169.254/opc/v1/instance/metadata/utility_subnet` 14 | cloudera_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cloudera_version` 15 | cloudera_major_version=`echo $cloudera_version | cut -d '.' -f1` 16 | cm_version=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_version` 17 | cm_major_version=`echo $cm_version | cut -d '.' -f1` 18 | worker_shape=`curl -L http://169.254.169.254/opc/v1/instance/metadata/worker_shape` 19 | worker_disk_count=`curl -L http://169.254.169.254/opc/v1/instance/metadata/block_volume_count` 20 | secure_cluster=`curl -L http://169.254.169.254/opc/v1/instance/metadata/secure_cluster` 21 | hdfs_ha=`curl -L http://169.254.169.254/opc/v1/instance/metadata/hdfs_ha` 22 | cluster_name=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cluster_name` 23 | cm_username=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_username` 24 | cm_password=`curl -L http://169.254.169.254/opc/v1/instance/metadata/cm_password` 25 | vcore_ratio=`curl -L http://169.254.169.254/opc/v1/instance/metadata/vcore_ratio` 26 | full_service_list=(ATLAS HBASE HDFS HIVE IMPALA KAFKA OOZIE RANGER SOLR SPARK_ON_YARN SQOOP_CLIENT YARN) 27 | service_list="ZOOKEEPER" 28 | rangeradmin_password='ChangeMe!' 29 | for service in ${full_service_list[@]}; do 30 | svc_check=`curl -L http://169.254.169.254/opc/v1/instance/metadata/svc_${service}` 31 | if [ $svc_check = "true" ]; then 32 | service_list=`echo -e "${service_list},${service}"` 33 | if [ $service = "RANGER" ]; then 34 | rangeradmin_password=`curl -L http://169.254.169.254/opc/v1/instance/metadata/rangeradmin_password` 35 | fi 36 | fi 37 | done; 38 | EXECNAME="TUNING" 39 | log "-> START" 40 | sed -i.bak 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config 41 | setenforce 0 42 | echo never | tee -a /sys/kernel/mm/transparent_hugepage/enabled 43 | echo "echo never | tee -a /sys/kernel/mm/transparent_hugepage/enabled" | tee -a /etc/rc.local 44 | echo vm.swappiness=1 | tee -a /etc/sysctl.conf 45 | echo 1 | tee /proc/sys/vm/swappiness 46 | echo net.ipv4.tcp_timestamps=0 >> /etc/sysctl.conf 47 | echo net.ipv4.tcp_sack=1 >> /etc/sysctl.conf 48 | echo net.core.rmem_max=4194304 >> /etc/sysctl.conf 49 | echo net.core.wmem_max=4194304 >> /etc/sysctl.conf 50 | echo net.core.rmem_default=4194304 >> /etc/sysctl.conf 51 | echo net.core.wmem_default=4194304 >> /etc/sysctl.conf 52 | echo net.core.optmem_max=4194304 >> /etc/sysctl.conf 53 | echo net.ipv4.tcp_rmem="4096 87380 4194304" >> /etc/sysctl.conf 54 | echo net.ipv4.tcp_wmem="4096 65536 4194304" >> /etc/sysctl.conf 55 | echo net.ipv4.tcp_low_latency=1 >> /etc/sysctl.conf 56 | sed -i "s/defaults 1 1/defaults,noatime 0 0/" /etc/fstab 57 | echo "hdfs - nofile 32768 58 | hdfs - nproc 2048 59 | hbase - nofile 32768 60 | hbase - nproc 2048" >> /etc/security/limits.conf 61 | ulimit -n 262144 62 | systemctl stop firewalld 63 | systemctl disable firewalld 64 | EXECNAME="KERBEROS" 65 | log "-> INSTALL" 66 | yum -y install krb5-server krb5-libs krb5-workstation >> $LOG_FILE 67 | KERBEROS_PASSWORD="SOMEPASSWORD" 68 | SCM_USER_PASSWORD="somepassword" 69 | kdc_fqdn=${cm_fqdn} 70 | realm="hadoop.com" 71 | REALM="HADOOP.COM" 72 | log "-> CONFIG" 73 | rm -f /etc/krb5.conf 74 | cat > /etc/krb5.conf << EOF 75 | # Configuration snippets may be placed in this directory as well 76 | includedir /etc/krb5.conf.d/ 77 | 78 | [libdefaults] 79 | default_realm = ${REALM} 80 | dns_lookup_realm = false 81 | dns_lookup_kdc = false 82 | rdns = false 83 | ticket_lifetime = 24h 84 | renew_lifetime = 7d 85 | forwardable = true 86 | udp_preference_limit = 1000000 87 | default_tkt_enctypes = rc4-hmac 88 | default_tgs_enctypes = rc4-hmac 89 | permitted_enctypes = rc4-hmac 90 | 91 | [realms] 92 | ${REALM} = { 93 | kdc = ${kdc_fqdn}:88 94 | admin_server = ${kdc_fqdn}:749 95 | default_domain = ${realm} 96 | } 97 | 98 | [domain_realm] 99 | .${realm} = ${REALM} 100 | ${realm} = ${REALM} 101 | bastion1.${cluster_domain} = ${REALM} 102 | .bastion1.${cluster_domain} = ${REALM} 103 | bastion2.${cluster_domain} = ${REALM} 104 | .bastion2.${cluster_domain} = ${REALM} 105 | bastion3.${cluster_domain} = ${REALM} 106 | .bastion3.${cluster_domain} = ${REALM} 107 | .public1.${cluster_domain} = ${REALM} 108 | public1.${cluster_domain} = ${REALM} 109 | .public2.${cluster_domain} = ${REALM} 110 | public2.${cluster_domain} = ${REALM} 111 | .public3.${cluster_domain} = ${REALM} 112 | public3.${cluster_domain} = ${REALM} 113 | .private1.${cluster_domain} = ${REALM} 114 | private1.${cluster_domain} = ${REALM} 115 | .private2.${cluster_domain} = ${REALM} 116 | private2.${cluster_domain} = ${REALM} 117 | .private3.${cluster_domain} = ${REALM} 118 | private3.${cluster_domain} = ${REALM} 119 | 120 | [kdc] 121 | profile = /var/kerberos/krb5kdc/kdc.conf 122 | 123 | [logging] 124 | kdc = FILE:/var/log/krb5kdc.log 125 | admin_server = FILE:/var/log/kadmin.log 126 | default = FILE:/var/log/krb5lib.log 127 | EOF 128 | rm -f /var/kerberos/krb5kdc/kdc.conf 129 | cat > /var/kerberos/krb5kdc/kdc.conf << EOF 130 | default_realm = ${REALM} 131 | 132 | [kdcdefaults] 133 | v4_mode = nopreauth 134 | kdc_ports = 0 135 | 136 | [realms] 137 | ${REALM} = { 138 | kdc_ports = 88 139 | admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab 140 | database_name = /var/kerberos/krb5kdc/principal 141 | acl_file = /var/kerberos/krb5kdc/kadm5.acl 142 | key_stash_file = /var/kerberos/krb5kdc/stash 143 | max_life = 10h 0m 0s 144 | max_renewable_life = 7d 0h 0m 0s 145 | master_key_type = des3-hmac-sha1 146 | supported_enctypes = rc4-hmac:normal 147 | default_principal_flags = +preauth 148 | } 149 | EOF 150 | rm -f /var/kerberos/krb5kdc/kadm5.acl 151 | cat > /var/kerberos/krb5kdc/kadm5.acl << EOF 152 | */admin@${REALM} * 153 | cloudera-scm@${REALM} * 154 | EOF 155 | kdb5_util create -r ${REALM} -s -P ${KERBEROS_PASSWORD} >> $LOG_FILE 156 | echo -e "addprinc root/admin\n${KERBEROS_PASSWORD}\n${KERBEROS_PASSWORD}\naddprinc cloudera-scm\n${SCM_USER_PASSWORD}\n${SCM_USER_PASSWORD}\nktadd -k /var/kerberos/krb5kdc/kadm5.keytab kadmin/admin\nktadd -k /var/kerberos/krb5kdc/kadm5.keytab kadmin/changepw\nexit\n" | kadmin.local -r ${REALM} 157 | log "-> START" 158 | systemctl start krb5kdc.service >> $LOG_FILE 159 | systemctl start kadmin.service >> $LOG_FILE 160 | systemctl enable krb5kdc.service >> $LOG_FILE 161 | systemctl enable kadmin.service >> $LOG_FILE 162 | EXECNAME="Cloudera Manager & Pre-Reqs Install" 163 | log "-> Installation" 164 | rpm --import https://archive.cloudera.com/cdh${cm_major_version}/${cm_version}/redhat7/yum//RPM-GPG-KEY-cloudera 165 | wget http://archive.cloudera.com/cm${cm_major_version}/${cm_version}/redhat7/yum/cloudera-manager.repo -O /etc/yum.repos.d/cloudera-manager.repo 166 | yum install cloudera-manager-server java-1.8.0-openjdk.x86_64 python-pip -y >> $LOG_FILE 167 | pip install psycopg2==2.7.5 --ignore-installed >> $LOG_FILE 168 | yum install oracle-j2sdk1.8.x86_64 cloudera-manager-daemons cloudera-manager-agent -y >> $LOG_FILE 169 | cp /etc/cloudera-scm-agent/config.ini /etc/cloudera-scm-agent/config.ini.orig 170 | sed -e "s/\(server_host=\).*/\1${cm_fqdn}/" -i /etc/cloudera-scm-agent/config.ini 171 | chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-agent/ 172 | systemctl start cloudera-scm-agent 173 | install_postgres(){ 174 | yum install postgreql-server -y 175 | EXECNAME="Postgresql Bootstrap" 176 | CURRENT_VERSION_MARKER='OCI_1' 177 | SLEEP_INTERVAL=5 178 | } 179 | stop_db() 180 | { 181 | systemctl stop postgresql 182 | } 183 | fail_or_continue() 184 | { 185 | local RET=$1 186 | local STR=$2 187 | if [[ $RET -ne 0 ]]; then 188 | stop_db 189 | if [[ -z $STR ]]; then 190 | STR="--> Error $RET" 191 | fi 192 | log "$STR, giving up" 193 | log "------- initialize-postgresql.sh failed -------" 194 | exit "$RET" 195 | fi 196 | } 197 | create_database() 198 | { 199 | local DB_CMD="sudo -u postgres psql" 200 | local DBNAME=$1 201 | local PW=$2 202 | local ROLE=$DBNAME 203 | if ! [ -z "$3" ];then 204 | local ROLE=$3 205 | echo "$3, $ROLE" 206 | fi 207 | echo "$ROLE" 208 | echo "CREATE ROLE $ROLE LOGIN PASSWORD '$PW';" 209 | $DB_CMD --command "CREATE ROLE $ROLE LOGIN PASSWORD '$PW';" 210 | fail_or_continue $? "Unable to create database role $ROLE" 211 | echo "CREATE DATABASE $DBNAME OWNER $ROLE;" 212 | $DB_CMD --command "CREATE DATABASE $DBNAME OWNER $ROLE;" 213 | fail_or_continue $? "Unable to create database $DBNAME" 214 | } 215 | db_exists() 216 | { 217 | grep -q -s -e "^$1$" "$DB_LIST_FILE" 218 | } 219 | create_random_password() 220 | { 221 | perl -le 'print map { ("a".."z", "A".."Z", 0..9)[rand 62] } 1..10' 222 | } 223 | create_scm_db() 224 | { 225 | if db_exists scm; then 226 | return 0 227 | fi 228 | 229 | local PW=$1 230 | create_database scm "$PW" 231 | 232 | orig_umask=$(umask) 233 | umask 0077 234 | echo "Creating SCM configuration file: $DB_PROP_FILE" 235 | cat > "$DB_PROP_FILE" << EOF 236 | com.cloudera.cmf.db.type=postgresql 237 | com.cloudera.cmf.db.host=localhost:$DB_PORT 238 | com.cloudera.cmf.db.name=scm 239 | com.cloudera.cmf.db.user=scm 240 | com.cloudera.cmf.db.password=$PW 241 | EOF 242 | 243 | umask "$orig_umask" 244 | fail_or_continue $? "Error creating file $DB_PROP_FILE" 245 | echo "Created db properties file $DB_PROP_FILE" 246 | echo scm >> "$DB_LIST_FILE" 247 | } 248 | create_hive_metastore() 249 | { 250 | local role='HIVEMETASTORESERVER' 251 | local db='metastore' 252 | local hive='hive' 253 | if db_exists $db; then 254 | return 0 255 | fi 256 | 257 | echo "Creating DB $db for role $role" 258 | local pw 259 | pw=$(create_random_password) 260 | create_database "$db" "$pw" "$hive" 261 | 262 | echo "host $db $hive 0.0.0.0/0 md5" >> "$DATA_DIR"/pg_hba.conf 263 | if [[ ! -f $MGMT_DB_PROP_FILE ]]; then 264 | orig_umask=$(umask) 265 | umask 0077 266 | touch $MGMT_DB_PROP_FILE 267 | umask "$orig_umask" 268 | fail_or_continue $? "Error creating file $MGMT_DB_PROP_FILE" 269 | fi 270 | local PREFIX="com.cloudera.cmf.$role.db" 271 | cat >> "$MGMT_DB_PROP_FILE" <> "$DATA_DIR"/pg_hba.conf 280 | echo "Created DB for role $role" 281 | echo "$db" >> "$DB_LIST_FILE" 282 | } 283 | create_mgmt_role_db() 284 | { 285 | local role=$1 286 | local db=$2 287 | if db_exists "$db"; then 288 | return 0 289 | fi 290 | echo "Creating DB $db for role $role" 291 | local pw 292 | pw=$(create_random_password) 293 | create_database "$db" "$pw" 294 | if [[ ! -f $MGMT_DB_PROP_FILE ]]; then 295 | orig_umask=$(umask) 296 | umask 0077 297 | touch $MGMT_DB_PROP_FILE 298 | umask "$orig_umask" 299 | fail_or_continue $? "Error creating file $MGMT_DB_PROP_FILE" 300 | fi 301 | local PREFIX="com.cloudera.cmf.$role.db" 302 | cat >> "$MGMT_DB_PROP_FILE" <> "$DATA_DIR"/pg_hba.conf 311 | log "-->Created DB for role $role" 312 | echo "$db" >> "$DB_LIST_FILE" 313 | } 314 | pg_hba_contains() 315 | { 316 | grep -q -s -e "^$1$" "$DATA_DIR"/pg_hba.conf 317 | } 318 | configure_remote_connections() 319 | { 320 | local FIRSTLINE="# block remote access for admin user" 321 | local SECONDLINE="host all postgres 0.0.0.0/0 reject" 322 | local THIRDLINE="# enable remote access for other users" 323 | local FOURTHLINE="host sameuser all 0.0.0.0/0 md5" 324 | if pg_hba_contains "$FIRSTLINE"; then 325 | return 0 326 | fi 327 | echo "$FIRSTLINE" >> "$DATA_DIR"/pg_hba.conf 328 | echo "$SECONDLINE" >> "$DATA_DIR"/pg_hba.conf 329 | echo "$THIRDLINE" >> "$DATA_DIR"/pg_hba.conf 330 | echo "$FOURTHLINE" >> "$DATA_DIR"/pg_hba.conf 331 | log "-->Enabled remote connections" 332 | } 333 | get_system_ram() 334 | { 335 | local free_output 336 | free_output=$(free -b | grep Mem) 337 | local regex="Mem:[[:space:]]+([[:digit:]]+)" 338 | if [[ $free_output =~ $regex ]]; then 339 | echo "${BASH_REMATCH[1]}" 340 | else 341 | fail_or_continue 1 "Unable to find amount of RAM on the system" 342 | fi 343 | } 344 | get_shared_buffers() 345 | { 346 | local ram 347 | ram=$(get_system_ram) 348 | local shmmax 349 | shmmax=$(cat /proc/sys/kernel/shmmax) 350 | local THIRTY_TWO_MB=$((32 * 1024 * 1024)) 351 | local EIGHT_GB=$((8 * 1024 * 1024 * 1024)) 352 | local SIXTEEN_GB=$((16 * 1024 * 1024 * 1024)) 353 | local shared_buffer; 354 | if [ ${#shmmax} -gt 11 ]; then 355 | shmmax=$SIXTEEN_GB 356 | fi 357 | if [ "$shmmax" -eq "$THIRTY_TWO_MB" ]; then 358 | let "shared_buffer=shmmax / 4" 359 | let "shared_buffer=shared_buffer / (8192 + 208)" 360 | echo "shared_buffers=$shared_buffer" 361 | elif [ "$shmmax" -gt "$THIRTY_TWO_MB" ]; then 362 | let "shared_buffer=shmmax / 2" 363 | if [ "$shared_buffer" -gt "$EIGHT_GB" ]; then 364 | shared_buffer=$EIGHT_GB 365 | fi 366 | let "quarter_of_ram=ram / 4" 367 | if [ "$shared_buffer" -gt "$quarter_of_ram" ]; then 368 | shared_buffer=$quarter_of_ram 369 | fi 370 | let "shared_buffer=shared_buffer / (8192 + 208)" 371 | echo "shared_buffers=$shared_buffer" 372 | fi 373 | } 374 | get_postgresql_major_version() 375 | { 376 | local psql_output 377 | psql_output=$(psql --version) 378 | local regex 379 | regex="^psql \(PostgreSQL\) ([[:digit:]]+)\..*" 380 | 381 | if [[ $psql_output =~ $regex ]]; then 382 | echo "${BASH_REMATCH[1]}" 383 | fi 384 | } 385 | get_standard_conforming_strings() 386 | { 387 | local psql_version 388 | psql_version=$(get_postgresql_major_version) 389 | if [[ $psql_version -gt 8 ]]; then 390 | echo "# This is needed to make Hive work with Postgresql 9.1 and above\\" 391 | echo "# See OPSAPS-11795\\" 392 | echo "standard_conforming_strings=off" 393 | fi 394 | } 395 | configure_postgresql_conf() 396 | { 397 | local CONF_FILE="$1" 398 | local IS_UPGRADE="$2" 399 | sed -e '/^listen_addresses\s*=/d' -i "$CONF_FILE" 400 | sed -e '/^max_connections\s*=/d' -i "$CONF_FILE" 401 | sed -e '/^shared_buffers\s*=/d' -i "$CONF_FILE" 402 | sed -e '/^standard_conforming_strings\s*=/d' -i "$CONF_FILE" 403 | local TMPFILE 404 | TMPFILE=$(mktemp /tmp/XXXXXXXX) 405 | cat "$CONF_FILE" >> "$TMPFILE" 406 | echo Adding configs 407 | sed -i "2a # === $CURRENT_VERSION_MARKER at $NOW" "$TMPFILE" 408 | sed -i "3a port = $DB_PORT" "$TMPFILE" 409 | sed -i "4a listen_addresses = '*'" "$TMPFILE" 410 | sed -i "5a max_connections = 500" "$TMPFILE" 411 | local LINE_NUM=6 412 | local SHARED_BUFFERS 413 | SHARED_BUFFERS="$(get_shared_buffers)" 414 | if [ -n "${SHARED_BUFFERS}" ]; then 415 | sed -i "${LINE_NUM}a ${SHARED_BUFFERS}" "$TMPFILE" 416 | LINE_NUM=7 417 | fi 418 | local SCS 419 | SCS="$(get_standard_conforming_strings)" 420 | if [ -n "${SCS}" ]; then 421 | sed -i "${LINE_NUM}a ${SCS}" "$TMPFILE" 422 | fi 423 | cat "$TMPFILE" > "$CONF_FILE" 424 | } 425 | wait_for_db_server_to_start() 426 | { 427 | log "Wait for DB server to start" 428 | i=0 429 | until [ $i -ge 5 ] 430 | do 431 | i=$((i+1)) 432 | sudo -u postgres psql -l && break 433 | sleep "${SLEEP_INTERVAL}" 434 | done 435 | if [ $i -ge 5 ]; then 436 | log "DB failed to start within $((i * SLEEP_INTERVAL)) seconds, exit with status 1" 437 | log "------- Postgresql startup failed -------" 438 | exit 1 439 | fi 440 | } 441 | install_postgres 442 | log "------- Begin Postgresql Setup -------" 443 | echo 'LC_ALL="en_US.UTF-8"' >> /etc/locale.conf 444 | log "-- Running Postgresql initdb --" 445 | su -l postgres -c "postgresql-setup initdb" 446 | log "-- Starting Postgresql --" 447 | systemctl start postgresql 448 | SCM_PWD=$(create_random_password) 449 | DATA_DIR=/var/lib/pgsql/data 450 | DB_HOST=$(hostname -f) 451 | DB_PORT=${DB_PORT:-5432} 452 | DB_HOSTPORT="$DB_HOST:$DB_PORT" 453 | DB_PROP_FILE=/etc/cloudera-scm-server/db.properties 454 | MGMT_DB_PROP_FILE=/etc/cloudera-scm-server/db.mgmt.properties 455 | DB_LIST_FILE=$DATA_DIR/scm.db.list 456 | NOW=$(date +%Y%m%d-%H%M%S) 457 | log "-- Configuring Postgresql --" 458 | configure_postgresql_conf $DATA_DIR/postgresql.conf 0 459 | echo "# Accept connections from all hosts" >> $DATA_DIR/pg_hba.conf 460 | sed -i '/host.*127.*ident/i \ 461 | host all all 127.0.0.1/32 md5 \ ' $DATA_DIR/pg_hba.conf 462 | /sbin/chkconfig postgresql on 463 | log "-- Restarting Postgresql --" 464 | systemctl restart postgresql 465 | wait_for_db_server_to_start 466 | log "-- Postgres DB Started --" 467 | log "-- Setting up SCM DB --" 468 | create_scm_db "$SCM_PWD" 469 | log "-- Setting up additional CM DBs --" 470 | create_mgmt_role_db ACTIVITYMONITOR amon 471 | create_mgmt_role_db REPORTSMANAGER rman 472 | create_mgmt_role_db NAVIGATOR nav 473 | create_mgmt_role_db NAVIGATORMETASERVER navms 474 | create_mgmt_role_db OOZIE oozie 475 | create_mgmt_role_db HUE hue 476 | create_mgmt_role_db SENTRY sentry 477 | log "-- Creating HIVE Metastore --" 478 | create_hive_metastore 479 | log "-- Running SCM DB Bootstrap --" 480 | /opt/cloudera/cm/schema/scm_prepare_database.sh postgresql scm scm "$SCM_PWD" >> "${LOG_FILE}" 2>&1 481 | log "-- Configuring Remote Connections --" 482 | configure_remote_connections 483 | log "-- Restarting Postgresql to refresh config --" 484 | systemctl restart postgresql 485 | wait_for_db_server_to_start 486 | log "-- DONE --" 487 | vol_match() { 488 | case $i in 489 | 1) disk="oraclevdb";; 490 | 2) disk="oraclevdc";; 491 | 3) disk="oraclevdd";; 492 | 4) disk="oraclevde";; 493 | 5) disk="oraclevdf";; 494 | 6) disk="oraclevdg";; 495 | 7) disk="oraclevdh";; 496 | 8) disk="oraclevdi";; 497 | 9) disk="oraclevdj";; 498 | 10) disk="oraclevdk";; 499 | 11) disk="oraclevdl";; 500 | 12) disk="oraclevdm";; 501 | 13) disk="oraclevdn";; 502 | 14) disk="oraclevdo";; 503 | 15) disk="oraclevdp";; 504 | 16) disk="oraclevdq";; 505 | 17) disk="oraclevdr";; 506 | 18) disk="oraclevds";; 507 | 19) disk="oraclevdt";; 508 | 20) disk="oraclevdu";; 509 | 21) disk="oraclevdv";; 510 | 22) disk="oraclevdw";; 511 | 23) disk="oraclevdx";; 512 | 24) disk="oraclevdy";; 513 | 25) disk="oraclevdz";; 514 | 26) disk="oraclevdab";; 515 | 27) disk="oraclevdac";; 516 | 28) disk="oraclevdad";; 517 | 29) disk="oraclevdae";; 518 | 30) disk="oraclevdaf";; 519 | 31) disk="oraclevdag";; 520 | esac 521 | } 522 | iscsi_setup() { 523 | log "-> ISCSI Volume Setup - Volume $i : IQN ${iqn[$n]}" 524 | iscsiadm -m node -o new -T ${iqn[$n]} -p 169.254.2.${n}:3260 525 | log "--> Volume ${iqn[$n]} added" 526 | iscsiadm -m node -o update -T ${iqn[$n]} -n node.startup -v automatic 527 | log "--> Volume ${iqn[$n]} startup set" 528 | iscsiadm -m node -T ${iqn[$n]} -p 169.254.2.${n}:3260 -l 529 | log "--> Volume ${iqn[$n]} done" 530 | } 531 | iscsi_target_only(){ 532 | log "-->Logging into Volume ${iqn[$n]}" 533 | iscsiadm -m node -T ${iqn[$n]} -p 169.254.2.${n}:3260 -l 534 | } 535 | EXECNAME="ISCSI" 536 | done="0" 537 | log "-- Detecting Block Volumes --" 538 | for i in `seq 2 33`; do 539 | if [ $done = "0" ]; then 540 | iscsiadm -m discoverydb -D -t sendtargets -p 169.254.2.${i}:3260 2>&1 2>/dev/null 541 | iscsi_chk=`echo -e $?` 542 | if [ $iscsi_chk = "0" ]; then 543 | iqn[$i]=`iscsiadm -m discoverydb -D -t sendtargets -p 169.254.2.${i}:3260 | gawk '{print $2}'` 544 | log "-> Discovered volume $((i-1)) - IQN: ${iqn[$i]}" 545 | continue 546 | else 547 | log "--> Discovery Complete - ${#iqn[@]} volumes found" 548 | done="1" 549 | fi 550 | fi 551 | done; 552 | log "-- Setup for ${#iqn[@]} Block Volumes --" 553 | if [ ${#iqn[@]} -gt 0 ]; then 554 | for i in `seq 1 ${#iqn[@]}`; do 555 | n=$((i+1)) 556 | iscsi_setup 557 | done; 558 | fi 559 | EXECNAME="DISK PROVISIONING" 560 | data_mount () { 561 | log "-->Mounting /dev/$disk to /data$dcount" 562 | mkdir -p /data$dcount 563 | mount -o noatime,barrier=1 -t ext4 /dev/$disk /data$dcount 564 | UUID=`lsblk -no UUID /dev/$disk` 565 | echo "UUID=$UUID /data$dcount ext4 defaults,noatime,discard,barrier=0 0 1" | tee -a /etc/fstab 566 | } 567 | block_data_mount () { 568 | log "-->Mounting /dev/oracleoci/$disk to /data$dcount" 569 | mkdir -p /data$dcount 570 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /data$dcount 571 | UUID=`lsblk -no UUID /dev/oracleoci/$disk` 572 | echo "UUID=$UUID /data$dcount ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 573 | } 574 | log "->Checking for disks..." 575 | dcount=0 576 | for disk in `cat /proc/partitions | grep nv`; do 577 | log "-->Processing /dev/$disk" 578 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/$disk 579 | data_mount 580 | dcount=$((dcount+1)) 581 | done; 582 | if [ ${#iqn[@]} -gt 0 ]; then 583 | for i in `seq 1 ${#iqn[@]}`; do 584 | n=$((i+1)) 585 | dsetup="0" 586 | while [ $dsetup = "0" ]; do 587 | vol_match 588 | log "-->Checking /dev/oracleoci/$disk" 589 | if [ -h /dev/oracleoci/$disk ]; then 590 | mke2fs -F -t ext4 -b 4096 -E lazy_itable_init=1 -O sparse_super,dir_index,extent,has_journal,uninit_bg -m1 /dev/oracleoci/$disk 591 | if [ $disk = "oraclevdb" ]; then 592 | log "--->Mounting /dev/oracleoci/$disk to /var/log/cloudera" 593 | mkdir -p /var/log/cloudera 594 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /var/log/cloudera 595 | UUID=`lsblk -no UUID /dev/oracleoci/$disk` 596 | echo "UUID=$UUID /var/log/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 597 | elif [ $disk = "oraclevdc" ]; then 598 | log "--->Mounting /dev/oracleoci/$disk to /opt/cloudera" 599 | if [ -d /opt/cloudera ]; then 600 | mv /opt/cloudera /opt/cloudera_pre 601 | mkdir -p /opt/cloudera 602 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /opt/cloudera 603 | mv /opt/cloudera_pre/* /opt/cloudera 604 | rm -fR /opt/cloudera_pre 605 | else 606 | mkdir -p /opt/cloudera 607 | mount -o noatime,barrier=1 -t ext4 /dev/oracleoci/$disk /opt/cloudera 608 | fi 609 | UUID=`lsblk -no UUID /dev/oracleoci/$disk` 610 | echo "UUID=$UUID /opt/cloudera ext4 defaults,_netdev,nofail,noatime,discard,barrier=0 0 2" | tee -a /etc/fstab 611 | else 612 | block_data_mount 613 | dcount=$((dcount+1)) 614 | fi 615 | /sbin/tune2fs -i0 -c0 /dev/oracleoci/$disk 616 | dsetup="1" 617 | else 618 | log "--->${disk} not found, running ISCSI setup again." 619 | iscsi_target_only 620 | sleep 5 621 | fi 622 | done; 623 | done; 624 | fi 625 | EXECNAME="Cloudera Manager" 626 | log "->Starting Cloudera Manager" 627 | chown -R cloudera-scm:cloudera-scm /etc/cloudera-scm-server 628 | systemctl start cloudera-scm-server 629 | EXECNAME="Cloudera Enterprise Data Hub" 630 | log "->Installing Python Pre-reqs" 631 | sudo yum install python python-pip -y >> $LOG_FILE 632 | sudo pip install --upgrade pip >> $LOG_FILE 633 | sudo pip install cm_client >> $LOG_FILE 634 | log "->Running Cluster Deployment" 635 | log "-->Host Discovery" 636 | detection_flag="0" 637 | w=1 638 | while [ $detection_flag = "0" ]; do 639 | worker_lookup=`host cloudera-worker-$w.${cluster_subnet}.${cluster_domain}` 640 | worker_check=`echo -e $?` 641 | if [ $worker_check = "0" ]; then 642 | worker_fqdn[$w]="cloudera-worker-$w.${cluster_subnet}.${cluster_domain}" 643 | w=$((w+1)) 644 | else 645 | detection_flag="1" 646 | fi 647 | done; 648 | fqdn_list="cloudera-utility-1.${utility_subnet}.${cluster_domain},cloudera-master-1.${cluster_subnet}.${cluster_domain},cloudera-master-2.${cluster_subnet}.${cluster_domain}" 649 | num_workers=${#worker_fqdn[@]} 650 | for w in `seq 1 $num_workers`; do 651 | fqdn_list=`echo "${fqdn_list},${worker_fqdn[$w]}"` 652 | done; 653 | log "-->Host List: ${fqdn_list}" 654 | log "-->Cluster Build" 655 | if [ $secure_cluster = "true" ]; then 656 | if [ $hdfs_ha = "true" ]; then 657 | log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres" 658 | python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres 2>&1 1>> $LOG_FILE 659 | else 660 | log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres" 661 | python /var/lib/cloud/instance/scripts/deploy_on_oci.py -S -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres 2>&1 1>> $LOG_FILE 662 | fi 663 | else 664 | if [ $hdfs_ha = "true" ]; then 665 | log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres" 666 | python /var/lib/cloud/instance/scripts/deploy_on_oci.py -H -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres 2>&1 1>> $LOG_FILE 667 | else 668 | log "---> python /var/lib/cloud/instance/scripts/deploy_on_oci.py -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres" 669 | python /var/lib/cloud/instance/scripts/deploy_on_oci.py -m ${cm_ip} -i ${fqdn_list} -d ${worker_disk_count} -w ${worker_shape} -n ${num_workers} -cdh ${cloudera_version} -N ${cluster_name} -a ${cm_username} -p ${cm_password} -v ${vcore_ratio} -C ${service_list} -R ${rangeradmin_password} -M postgres 2>&1 1>> $LOG_FILE 670 | fi 671 | fi 672 | log "->DONE" 673 | -------------------------------------------------------------------------------- /scripts/exportBDC.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. 4 | # 5 | # Utility script to export user data from Oracle Big Data Service. 6 | # This script exports Hive Metadata, Zeppelin Notebooks, Service Configuration and Version data 7 | # to a tar file. 8 | # 9 | # This must be run as root user on Ambari Host. Hive and Zeppelin Serivce must be in 10 | # stopped state, otherwise, script will exit. 11 | # 12 | # Usage - exportBDC.py 13 | # Run this script on Ambari host as root user. 14 | # 15 | 16 | import json 17 | import urllib2, base64 18 | import os, tarfile, shutil, glob, socket, sys, subprocess 19 | from contextlib import closing 20 | import datetime, logging as log 21 | 22 | if(len(sys.argv) < 2): 23 | log.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', stream=sys.stdout, level=log.INFO) 24 | log.error("Usage: exportBDC.py [-v]") 25 | log.error("Run this script on Ambari host as root user.") 26 | log.error("Use -v for more detailed log") 27 | sys.exit(0) 28 | 29 | if("-v" in sys.argv): 30 | log.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', stream=sys.stdout, level=log.DEBUG) 31 | else: 32 | log.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', stream=sys.stdout, level=log.INFO) 33 | 34 | ambari_ip = socket.gethostname() 35 | config_path = sys.argv[1] 36 | 37 | FNULL = open(os.devnull, 'w') 38 | # FNULL = subprocess.STDOUT # This helps in debugging hive export issue 39 | 40 | ambari_port = '8080' 41 | ambari_url = 'https://' + ambari_ip + ':' + ambari_port + '/api/v1/clusters/' 42 | 43 | config_json = json.loads("{}") 44 | output_path = "" 45 | cluster_name = "" 46 | services = [] 47 | components = {} 48 | temp_config_tars = "" 49 | temp_extract_path = "" 50 | final_tarball = "" 51 | 52 | 53 | def readConfig(): 54 | global config_json 55 | if os.path.exists(config_path): 56 | with open(config_path) as data_file: 57 | config_json = json.load(data_file) 58 | else: 59 | log.error("Config file, " + config_path + " not found...") 60 | sys.exit(0) 61 | 62 | 63 | def loadConfig(): 64 | global output_path, cluster_name, temp_extract_path, final_tarball, services 65 | 66 | output_path = config_json["export_dir"] 67 | os.system('mkdir -p ' + output_path) 68 | 69 | cluster_name = getClusterName() 70 | log.debug("Cluster Name - " + cluster_name) 71 | temp_extract_path = output_path + "/" + cluster_name 72 | if(os.path.exists(temp_extract_path)): 73 | shutil.rmtree(temp_extract_path) 74 | os.mkdir(temp_extract_path) 75 | 76 | now = datetime.datetime.now() 77 | timestamp = now.strftime('%d_%b_%Y_%H_%M_%S') 78 | final_tarball = """%s/export_%s_%s.tar.gz"""%(output_path, cluster_name, timestamp) 79 | services = getServices() 80 | log.debug("List of services found in the cluster - " + ','.join(map(str, services))) 81 | 82 | log.info("Exporting Oracle Big Data Cloud Service : " + cluster_name) 83 | log.info("This may take a few minutes to complete.\n") 84 | 85 | def ambariApiRes(url): 86 | base64string = base64.encodestring('%s:%s' % (config_json["ambari_username"], config_json["ambari_password"])).replace('\n', '') 87 | req = urllib2.Request(url) 88 | req.add_header('X-Requested-By', 'ambari') 89 | req.add_header("Authorization", "Basic %s" % base64string) 90 | 91 | try: 92 | response = urllib2.urlopen(req).read() 93 | except urllib2.HTTPError, e: 94 | log.debug("Ambari Rest Api failed for - " + url) 95 | response = "{}" 96 | 97 | responseJson = json.loads(response) 98 | return responseJson 99 | 100 | 101 | def isServiceStopped(service_name): 102 | url = """%s%s/services/%s?fields=ServiceInfo/state"""%(ambari_url, cluster_name, service_name) 103 | 104 | log.debug("""Url to check the status of service, %s - %s"""%(service_name, url)) 105 | 106 | base64string = base64.encodestring('%s:%s' % (config_json["ambari_username"], config_json["ambari_password"])).replace('\n', '') 107 | req = urllib2.Request(url) 108 | req.add_header('X-Requested-By', 'ambari') 109 | req.add_header("Authorization", "Basic %s" % base64string) 110 | 111 | response = urllib2.urlopen(req).read() 112 | responseJson = json.loads(response) 113 | 114 | if(responseJson["ServiceInfo"]["state"] == "INSTALLED"): 115 | return True 116 | else: 117 | return False 118 | 119 | 120 | def preCheck(): 121 | servicesToBeChecked = ["HIVE", "ZEPPELIN"] 122 | 123 | log.debug("Performing ") 124 | for service in servicesToBeChecked: 125 | # Checking if a service is stopped 126 | if not (isServiceStopped(service)): 127 | log.error("""%s service is not in stopped state in Ambari. Please stop it using Ambari and rerun"""%(service)) 128 | sys.exit(0) 129 | 130 | 131 | def ambari_config_download(url): 132 | # log.info("Config url --- " + url) 133 | base64string = base64.encodestring('%s:%s' % (config_json["ambari_username"], config_json["ambari_password"])).replace('\n', '') 134 | req = urllib2.Request(url) 135 | req.add_header('X-Requested-By', 'ambari') 136 | req.add_header("Authorization", "Basic %s" % base64string) 137 | 138 | try: 139 | # response = urllib2.urlopen(req, context=ctx) 140 | response = urllib2.urlopen(req) 141 | except urllib2.HTTPError, e: 142 | response = None 143 | 144 | return response 145 | 146 | 147 | def getClusterName(): 148 | url = ambari_url 149 | responseJson = ambariApiRes(url) 150 | 151 | return responseJson["items"][0]['Clusters']['cluster_name'] 152 | 153 | 154 | def getServices(): 155 | url = ambari_url + cluster_name + '/services' 156 | responseJson = ambariApiRes(url) 157 | for item in responseJson['items']: 158 | services.append(item['ServiceInfo']['service_name']) 159 | return services 160 | 161 | 162 | def populateComponents(): 163 | for service in services: 164 | # log.info("Getting components for service, " + service) 165 | url = ambari_url + cluster_name + '/services/' + service + '/components' 166 | responseJson = ambariApiRes(url) 167 | for item in responseJson["items"]: 168 | if components.has_key(service): 169 | components[service].append(item["ServiceComponentInfo"]["component_name"]) 170 | else: 171 | components[service] = [] 172 | components[service].append(item["ServiceComponentInfo"]["component_name"]) 173 | return components 174 | 175 | 176 | def downloadFile(fileName, resp): 177 | with open(fileName, "w") as local_file: 178 | local_file.write(resp.read()) 179 | 180 | 181 | def getConfigs(): 182 | global temp_config_tars 183 | # Cleaning up before downloading the configs 184 | # log.info("Cleaning up before downloading the configs...") 185 | temp_config_tars = output_path + "/config_tars/" 186 | if (os.path.isdir(temp_config_tars)): 187 | shutil.rmtree(temp_config_tars) 188 | os.mkdir(temp_config_tars) 189 | 190 | for service in components: 191 | for component in components[service]: 192 | # log.info("Getting config for service, " + service + " & component, " + component) 193 | url = ambari_url + cluster_name + '/services/' + service + '/components/' + component + "?format=client_config_tar" 194 | resp = ambari_config_download(url) 195 | fileName = temp_config_tars + "/" + component + "-configs.tar.gz" 196 | if(resp != None): 197 | downloadFile(fileName, resp) 198 | log.debug("Configuration is downloaded to " + fileName + " ...") 199 | else: 200 | log.debug("No config found for service, " + service + " & component, " + component) 201 | 202 | 203 | def prepareForPackaging(): 204 | temp_configs_path = temp_extract_path + "/" + "config" 205 | if(os.path.exists(temp_configs_path)): 206 | shutil.rmtree(temp_configs_path) 207 | os.mkdir(temp_configs_path) 208 | for file in glob.glob(temp_config_tars + "/*.tar.gz"): 209 | name = os.path.basename(file).split("-configs.tar.gz")[0] 210 | tf = tarfile.open(file) 211 | tf.extractall(path=temp_configs_path + "/" + name) 212 | tf.close() 213 | # Delete the temp config tars directory 214 | if(os.path.exists(temp_config_tars)): 215 | shutil.rmtree(temp_config_tars) 216 | 217 | 218 | def package(): 219 | log.debug("Creating the target tarball, " + final_tarball) 220 | with closing(tarfile.open(final_tarball, "w:gz")) as tar: 221 | tar.add(temp_extract_path, arcname='.') 222 | 223 | 224 | def cleanup(): 225 | log.debug("Perform final cleanup...") 226 | shutil.rmtree(temp_extract_path) 227 | 228 | 229 | def backupHDPConfigs(): 230 | log.info("") 231 | printDottedLine() 232 | log.info("Configuration") 233 | printDottedLine() 234 | log.info("Exporting Service Configuration data ....") 235 | populateComponents() 236 | getConfigs() 237 | prepareForPackaging() 238 | log.info("Completed exporting Exporting Service Configuration data.") 239 | 240 | 241 | def getVersions(): 242 | log.info("") 243 | printDottedLine() 244 | log.info("Stack component versions") 245 | printDottedLine() 246 | log.info("Exporting stack component versions....") 247 | services_list = ",".join(services) 248 | versions = "" 249 | version_file_path = temp_extract_path + "/stack" 250 | version_file = version_file_path + "/StackVersions.txt" 251 | if(os.path.isdir(version_file_path)): 252 | shutil.rmtree(version_file_path) 253 | os.mkdir(version_file_path) 254 | temp_file = temp_extract_path + "/StackVersions_temp" 255 | 256 | command=""" curl -o %s -u %s:%s -1 -s -k 'https://%s:%s/api/v1/stacks/HDP/versions/2.4/services?StackServices/service_name.in(%s)&fields=StackServices/*' """%(temp_file, config_json["ambari_username"], config_json["ambari_password"], ambari_ip,ambari_port, services_list) 257 | log.debug("Generated command to get the stack versions, " + command) 258 | subprocess.call(command, shell=True) 259 | 260 | f = open(temp_file, "r") 261 | res = f.read() 262 | 263 | responseJson = json.loads(res) 264 | for service in responseJson["items"]: 265 | versions = versions + service["StackServices"]["service_name"] + " : " + service["StackServices"]["service_version"] + "\n" 266 | 267 | f = open(version_file, "w") 268 | f.write(versions) 269 | log.debug("Cleaning temporary files created for Stack component versions Export...") 270 | if(os.path.exists(temp_file)): 271 | os.remove(temp_file) 272 | log.info("Completed exporting stack component versions.") 273 | 274 | 275 | def backupZeppelinNotes(): 276 | log.info("") 277 | printDottedLine() 278 | log.info("Zeppelin Notebooks") 279 | printDottedLine() 280 | log.info("Exporting Zeppelin Notebooks....") 281 | temp_zeppelin_notes = temp_extract_path + "/zeppelin/notebook" 282 | if (os.path.isdir(temp_zeppelin_notes)): 283 | shutil.rmtree(temp_zeppelin_notes) 284 | # The command below creates Zeppelin_Notebooks in hdfs home directory 285 | if (os.path.isdir("/var/lib/hadoop-hdfs/notebook")): 286 | shutil.rmtree("/var/lib/hadoop-hdfs/notebook") 287 | 288 | log.debug("Taking the zeppelin notebooks from hdfs://user/zeppelin/notebook notebook") 289 | command = "su - hdfs -c 'hdfs dfs -copyToLocal /user/zeppelin/notebook notebook'" 290 | subprocess.call(command, shell=True) 291 | 292 | log.debug("Cleaning temporary files created for Zeppelin Notebook Export...") 293 | shutil.copytree("/var/lib/hadoop-hdfs/notebook", temp_zeppelin_notes) 294 | shutil.rmtree("/var/lib/hadoop-hdfs/notebook") 295 | log.info("Completed exporting Zeppelin Notebooks.") 296 | 297 | 298 | def getHiveMetaDBName(): 299 | lookup = "ambari.hive.db.schema.name" 300 | url = """%s%s/configurations/service_config_versions?service_name=HIVE"""%(ambari_url, cluster_name) 301 | log.debug("Url to get the hive metastore db name - " + url) 302 | 303 | try: 304 | response_json = ambariApiRes(url) 305 | for config in response_json["items"]: 306 | if (config["is_current"] == True): 307 | for configuration in config["configurations"]: 308 | if lookup in configuration["properties"]: 309 | log.debug("Hive metastore DBName is - " + configuration["properties"][lookup]) 310 | return configuration["properties"][lookup] 311 | except: 312 | log.error("Failed to get hive metastore db name from Ambari. hive is the default metastore db name") 313 | # On failing to find return hive as default 314 | return "hive" 315 | 316 | 317 | def backupHiveMetadata(): 318 | log.info("") 319 | printDottedLine() 320 | log.info("Hive metadata") 321 | printDottedLine() 322 | log.info("Exporting Hive metadata....") 323 | 324 | hive_metastore_db = getHiveMetaDBName() 325 | 326 | if (os.path.isdir(temp_extract_path + "/hive_metadata")): 327 | shutil.rmtree(temp_extract_path + "/hive_metadata") 328 | os.mkdir(temp_extract_path + "/hive_metadata") 329 | temp_extract_hive_file = temp_extract_path + "/hive_metadata/hive_metadata_dump.sql" 330 | command="""mysqldump %s > %s"""%(hive_metastore_db, temp_extract_hive_file) 331 | subprocess.call(command, shell=True) 332 | 333 | log.info("Completed exporting Hive metadata.") 334 | 335 | def printDottedLine(): 336 | log.info("-------------------------------------------------------") 337 | 338 | 339 | log.info("") 340 | printDottedLine() 341 | log.info("Utility to export metadata from Big Data Cloud Service") 342 | printDottedLine() 343 | log.info("") 344 | 345 | readConfig() 346 | loadConfig() 347 | 348 | preCheck() 349 | 350 | backupHDPConfigs() 351 | backupZeppelinNotes() 352 | backupHiveMetadata() 353 | getVersions() 354 | 355 | package() 356 | cleanup() 357 | log.info("") 358 | log.info("") 359 | log.info("""Completed export from Oracle Big Data Cloud Service : %s to %s."""%(cluster_name, final_tarball)) 360 | -------------------------------------------------------------------------------- /scripts/my.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | datadir=/var/lib/mysql 3 | socket=/var/lib/mysql/mysql.sock 4 | transaction-isolation = READ-COMMITTED 5 | # Disabling symbolic-links is recommended to prevent assorted security risks; 6 | # to do so, uncomment this line: 7 | symbolic-links = 0 8 | log-error=/var/log/mysqld.log 9 | 10 | key_buffer_size = 32M 11 | max_allowed_packet = 16M 12 | thread_stack = 256K 13 | thread_cache_size = 64 14 | 15 | max_connections = 550 16 | #expire_logs_days = 10 17 | #max_binlog_size = 100M 18 | 19 | #log_bin should be on a disk with enough free space. 20 | #Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your 21 | #system and chown the specified folder to the mysql user. 22 | log_bin=/var/lib/mysql/mysql_binary_log 23 | 24 | #In later versions of MySQL, if you enable the binary log and do not set 25 | #a server_id, MySQL will not start. The server_id must be unique within 26 | #the replicating group. 27 | server_id=1 28 | 29 | binlog_format = mixed 30 | 31 | read_buffer_size = 2M 32 | read_rnd_buffer_size = 16M 33 | sort_buffer_size = 8M 34 | join_buffer_size = 8M 35 | 36 | # InnoDB settings 37 | innodb_file_per_table = 1 38 | innodb_flush_log_at_trx_commit = 2 39 | innodb_log_buffer_size = 64M 40 | innodb_buffer_pool_size = 4G 41 | innodb_thread_concurrency = 8 42 | innodb_flush_method = O_DIRECT 43 | innodb_log_file_size = 512M 44 | 45 | [mysqld_safe] 46 | log-error=/var/log/mysqld.log 47 | pid-file=/var/run/mysqld/mysqld.pid 48 | 49 | sql_mode=STRICT_ALL_TABLES 50 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # SSH Keys - Put this to top level because they are required 3 | # --------------------------------------------------------------------------------------------------------------------- 4 | 5 | variable "ssh_provided_key" { 6 | default = "" 7 | } 8 | 9 | # --------------------------------------------------------------------------------------------------------------------- 10 | # Network Settings 11 | # --------------------------------------------------------------------------------------------------------------------- 12 | variable "useExistingVcn" { 13 | default = "false" 14 | } 15 | 16 | variable "hide_public_subnet" { 17 | default = "true" 18 | } 19 | 20 | variable "hide_private_subnet" { 21 | default = "true" 22 | } 23 | 24 | variable "custom_cidrs" { 25 | default = "false" 26 | } 27 | 28 | variable "VPC_CIDR" { 29 | default = "10.0.0.0/16" 30 | } 31 | 32 | variable "edge_cidr" { 33 | default = "10.0.1.0/24" 34 | } 35 | 36 | variable "public_cidr" { 37 | default = "10.0.2.0/24" 38 | } 39 | 40 | variable "private_cidr" { 41 | default = "10.0.3.0/24" 42 | } 43 | 44 | variable "blockvolume_cidr" { 45 | default = "10.0.4.0/24" 46 | } 47 | 48 | variable "myVcn" { 49 | default = " " 50 | } 51 | 52 | variable "clusterSubnet" { 53 | default = " " 54 | } 55 | 56 | variable "bastionSubnet" { 57 | default = " " 58 | } 59 | 60 | variable "utilitySubnet" { 61 | default = " " 62 | } 63 | 64 | variable "blockvolumeSubnet" { 65 | default = " " 66 | } 67 | 68 | variable "vcn_dns_label" { 69 | default = "clouderavcn" 70 | } 71 | 72 | variable "secondary_vnic_count" { 73 | default = "0" 74 | } 75 | 76 | variable "blockvolume_subnet_id" { 77 | default = " " 78 | } 79 | 80 | variable "worker_domain" { 81 | default = " " 82 | } 83 | 84 | # --------------------------------------------------------------------------------------------------------------------- 85 | # ORM Schema variables 86 | # You should modify these based on deployment requirements. 87 | # These default to recommended values 88 | # --------------------------------------------------------------------------------------------------------------------- 89 | 90 | variable "meta_db_type" { 91 | default = "mysql" 92 | } 93 | 94 | variable "use_edge_nodes" { 95 | default = "false" 96 | } 97 | 98 | variable "enable_block_volumes" { 99 | default = "true" 100 | } 101 | 102 | variable "cm_username" { 103 | default = "cm_admin" 104 | } 105 | 106 | variable "cm_password" { 107 | default = "changeme" 108 | } 109 | 110 | variable "provide_ssh_key" { 111 | default = "true" 112 | } 113 | 114 | variable "vcore_ratio" { 115 | default = "2" 116 | } 117 | 118 | variable "yarn_scheduler" { 119 | default = "capacity" 120 | } 121 | 122 | variable "enable_secondary_vnic" { 123 | default = "false" 124 | } 125 | 126 | # --------------------------------------------------------------------------------------------------------------------- 127 | # Cloudera variables 128 | # You should modify these based on deployment requirements. 129 | # These default to recommended minimum values in most cases 130 | # --------------------------------------------------------------------------------------------------------------------- 131 | 132 | # Cloudera Manager Version 133 | variable "cm_version" { 134 | default = "7.4.4" 135 | } 136 | # Cloudera Enterprise Data Hub Version 137 | variable "cloudera_version" { 138 | default = "7.1.7.0" 139 | } 140 | variable "secure_cluster" { 141 | default = "True" 142 | } 143 | 144 | variable "hdfs_ha" { 145 | default = "False" 146 | } 147 | 148 | variable "worker_instance_shape" { 149 | default = "VM.Standard2.16" 150 | } 151 | 152 | variable "worker_node_count" { 153 | default = "5" 154 | } 155 | 156 | variable "data_blocksize_in_gbs" { 157 | default = "700" 158 | } 159 | 160 | variable "block_volumes_per_worker" { 161 | default = "3" 162 | } 163 | 164 | variable "customize_block_volume_performance" { 165 | default = "false" 166 | } 167 | 168 | variable "block_volume_high_performance" { 169 | default = "false" 170 | } 171 | 172 | variable "block_volume_cost_savings" { 173 | default = "false" 174 | } 175 | 176 | variable "vpus_per_gb" { 177 | default = "10" 178 | } 179 | 180 | variable "utility_instance_shape" { 181 | default = "VM.Standard2.8" 182 | } 183 | 184 | variable "master_instance_shape" { 185 | default = "VM.Standard2.8" 186 | } 187 | 188 | variable "master_node_count" { 189 | default = "2" 190 | } 191 | 192 | # Size for Cloudera Log Volumes across all hosts deployed to /var/log/cloudera 193 | 194 | variable "log_volume_size_in_gbs" { 195 | default = "200" 196 | } 197 | 198 | # Size for Volume across all hosts deployed to /opt/cloudera 199 | 200 | variable "cloudera_volume_size_in_gbs" { 201 | default = "300" 202 | } 203 | 204 | # Size for NameNode and SecondaryNameNode data volume (Journal Data) 205 | 206 | variable "nn_volume_size_in_gbs" { 207 | default = "500" 208 | } 209 | 210 | variable "bastion_instance_shape" { 211 | default = "VM.Standard2.4" 212 | } 213 | 214 | variable "bastion_node_count" { 215 | default = "1" 216 | } 217 | 218 | # Which AD to target - this can be adjusted. Default 1 for single AD regions. 219 | variable "availability_domain" { 220 | default = "1" 221 | } 222 | 223 | variable "cluster_name" { 224 | default = "TestCluster" 225 | } 226 | 227 | variable "objectstoreRAID" { 228 | default = "false" 229 | } 230 | 231 | variable "AdvancedOptions" { 232 | default = "false" 233 | } 234 | 235 | variable "svc_ATLAS" { 236 | default = "false" 237 | } 238 | 239 | variable "svc_HBASE" { 240 | default = "true" 241 | } 242 | 243 | variable "svc_HDFS" { 244 | default = "true" 245 | } 246 | 247 | variable "svc_HIVE" { 248 | default = "true" 249 | } 250 | 251 | variable "svc_IMPALA" { 252 | default = "true" 253 | } 254 | 255 | variable "svc_KAFKA" { 256 | default = "true" 257 | } 258 | 259 | variable "svc_OOZIE" { 260 | default = "true" 261 | } 262 | 263 | variable "svc_RANGER" { 264 | default = "false" 265 | } 266 | 267 | variable "svc_SOLR" { 268 | default = "true" 269 | } 270 | 271 | variable "svc_SPARK_ON_YARN" { 272 | default = "true" 273 | } 274 | 275 | variable "svc_SQOOP_CLIENT" { 276 | default = "true" 277 | } 278 | 279 | variable "svc_YARN" { 280 | default = "true" 281 | } 282 | 283 | variable "rangeradmin_password" { 284 | default = "Test123!" 285 | } 286 | 287 | variable "enable_debug" { 288 | default = "false" 289 | } 290 | # --------------------------------------------------------------------------------------------------------------------- 291 | # Environmental variables 292 | # You probably want to define these as environmental variables. 293 | # Instructions on that are here: https://github.com/oracle/oci-quickstart-prerequisites 294 | # --------------------------------------------------------------------------------------------------------------------- 295 | 296 | variable "compartment_ocid" {} 297 | 298 | # Required by the OCI Provider 299 | 300 | variable "tenancy_ocid" {} 301 | variable "region" {} 302 | 303 | # --------------------------------------------------------------------------------------------------------------------- 304 | # Constants 305 | # You probably don't need to change these. 306 | # --------------------------------------------------------------------------------------------------------------------- 307 | 308 | // See https://docs.cloud.oracle.com/en-us/iaas/images/image/f53b2e1c-a21c-41ab-96b1-18896bdec16f/ 309 | // Oracle-provided image "CentOS-7-2019.07.18-0" 310 | // Kernel Version: 3.10.0-957.21.3.el7.x86_64 311 | variable "CentOSImageOCID" { 312 | type = "map" 313 | default = { 314 | ap-mumbai-1 = "ocid1.image.oc1.ap-mumbai-1.aaaaaaaaojdjmlt7hhhyu6ev77fptrpcjza2elnhubmhauxx7ik53g3k4clq" 315 | ap-seoul-1 = "ocid1.image.oc1.ap-seoul-1.aaaaaaaa2liqaihg2b3dlxl54zqyt7zjvmxdunp6buivbtqhhvurnpepbvta" 316 | ap-tokyo-1 = "ocid1.image.oc1.ap-tokyo-1.aaaaaaaa7cjkigefv2b3hi32ku4yhwvbtlbn6ektgy25xuopekbcfltequxq" 317 | ca-toronto-1 = "ocid1.image.oc1.ca-toronto-1.aaaaaaaapgumj7xlcpfqugii7i7y722rfaib7xsc4tnoeikwwtsrrqxzf5qq" 318 | eu-frankfurt-1 = "ocid1.image.oc1.eu-frankfurt-1.aaaaaaaassfjfflfwty6c24gxoou224djh7rfm3cdnnq5v2jcx6eslwx4fpa" 319 | eu-zurich-1 = "ocid1.image.oc1.eu-zurich-1.aaaaaaaaqsi7yuqw7jk3wslena3lvpaxrtzpvoz7kelomvpwpdly7me3sixq" 320 | sa-saopaulo-1 = "ocid1.image.oc1.sa-saopaulo-1.aaaaaaaaanlyfas5floju6eiggf5jqh5oxsaoyjtlziaygabwnklh3gypfva" 321 | uk-london-1 = "ocid1.image.oc1.uk-london-1.aaaaaaaajyqa7buxw3jkgs5krmxmlnsek24dpby52scb7wsfln55cixusooa" 322 | us-ashburn-1 = "ocid1.image.oc1.iad.aaaaaaaatp4lxfmhmzebvfsw54tttkiv4jarrohqykbtmee5x2smxlqnr75a" 323 | us-phoenix-1 = "ocid1.image.oc1.phx.aaaaaaaava2go3l5jvj2ypu6poqgvhzypdwg6qbhkcs5etxewvulgizxy6fa" 324 | } 325 | } 326 | 327 | 328 | // See https://docs.oracle.com/en-us/iaas/images/image/7e0c4acd-2642-4f65-96ff-8c820f206d09/ 329 | // Oracle-provided image "Oracle-Linux-7.9-2021.10.04-0" 330 | 331 | variable "OELImageOCID" { 332 | type = "map" 333 | default = { 334 | ap-chuncheon-1 = "ocid1.image.oc1.ap-chuncheon-1.aaaaaaaanfughyhlrkcwluddtuisqp5idqhoz2e6yrsmnaxk3b7adlneizkq" 335 | ap-hyderabad-1 = "ocid1.image.oc1.ap-hyderabad-1.aaaaaaaabdukecj2l3hkplgu3zq7cja7pjqdiiu4vtltzyjwkojw3mqhtfya" 336 | ap-melbourne-1 = "ocid1.image.oc1.ap-melbourne-1.aaaaaaaaz5bdk2vrwlghroyfmsw22nqso52dsijzfvwyj2jthvekksbtmyra" 337 | ap-mumbai-1 = "ocid1.image.oc1.ap-mumbai-1.aaaaaaaagwbhu75joxgdn6cwilubuftqiutuy33xxu77c6zemtfldm23eslq" 338 | ap-osaka-1 = "ocid1.image.oc1.ap-osaka-1.aaaaaaaayq5v6m7qddhnwmdzav3noqanyrssokijolnjwgbewumgno5dg3ra" 339 | ap-seoul-1 = "ocid1.image.oc1.ap-seoul-1.aaaaaaaa5xfn4bmyde3rpmavpmyordhe6rs4siuquwigkwrtbszlkrjqp45q" 340 | ap-sydney-1 = "ocid1.image.oc1.ap-sydney-1.aaaaaaaaz5me7ycjpe6orq4xfgbfsvqwrc24ozhhvlpwbophf2qo7fiya44q" 341 | ap-tokyo-1 = "ocid1.image.oc1.ap-tokyo-1.aaaaaaaajns45jevzrv2ibtn6utyzpqr4oblqiwylzzmc53cshhcsvnt6aha" 342 | ca-montreal-1 = "ocid1.image.oc1.ca-montreal-1.aaaaaaaaciwparrvtjlonsx2dpzgpb3s27vegtczuw5ww3by6d3sm67nwdxa" 343 | ca-toronto-1 = "ocid1.image.oc1.ca-toronto-1.aaaaaaaa3hkkn5phegzsh3ikli5ju5ymygp6rqvl7e2ippjbrvtzoooeeumq" 344 | eu-amsterdam-1 = "ocid1.image.oc1.eu-amsterdam-1.aaaaaaaaufw7ml3y5m46miwihosq7h72o3n6kwndggqzz7edzwbun3ok3ypq" 345 | eu-frankfurt-1 = "ocid1.image.oc1.eu-frankfurt-1.aaaaaaaanw7ldk5drfpfqtianagutbipglptmu43iwanj2rmdxkqowecvcwa" 346 | eu-zurich-1 = "ocid1.image.oc1.eu-zurich-1.aaaaaaaa5rdszqyu25mqdnxxgluqeir2xrn2up75o227tnqk6zlp23ly325q" 347 | me-dubai-1 = "ocid1.image.oc1.me-dubai-1.aaaaaaaahautqpyy47fq4r6ekyzowd623icsat7bl2ya7dxg6dtdl6i424vq" 348 | me-jeddah-1 = "ocid1.image.oc1.me-jeddah-1.aaaaaaaa4fijw22qksb2xyfmovnoxwp2asxke5ppcg37iqdc6tb43lui2ipa" 349 | sa-santiago-1 = "ocid1.image.oc1.sa-santiago-1.aaaaaaaajejucvyg7jrsnvdp4jrdxybcfk7wvgn56t544ugh75uuuyor34ha" 350 | sa-saopaulo-1 = "ocid1.image.oc1.sa-saopaulo-1.aaaaaaaamka2yowr6jcoqdpl5gbaveal4gl3llfqfordz5zqvf2djts4yxra" 351 | sa-vinhedo-1 = "ocid1.image.oc1.sa-vinhedo-1.aaaaaaaala3gk3rn677mbqxx7s6plka5aztx6msjuzvb6s73wrhe3ckl6zja" 352 | uk-cardiff-1 = "ocid1.image.oc1.uk-cardiff-1.aaaaaaaavm2td6uswb4mskicwehtznstk36regjn4out5ywqksrekmnepuva" 353 | uk-london-1 = "ocid1.image.oc1.uk-london-1.aaaaaaaaqetdbotutgsdclw3ainkbcql33qwood6tj4yhq6xfscthtc6akqq" 354 | us-ashburn-1 = "ocid1.image.oc1.iad.aaaaaaaakjvsts7rf7umrlqtw5hbhc3gjotadu7thfn5cfathwdn3awht7ca" 355 | us-phoenix-1 = "ocid1.image.oc1.phx.aaaaaaaahmjqvzmd4aja4q5c47ustlgsw23j6afwhsqe5o354bnmdlgleaza" 356 | us-sanjose-1 = "ocid1.image.oc1.us-sanjose-1.aaaaaaaadthwf7yo6unsgqborhu5yw4klpr5i6ee4j6ipjfablz5kqrbvata" 357 | } 358 | } 359 | 360 | -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12" 3 | } 4 | --------------------------------------------------------------------------------