├── LICENSE ├── README.md ├── clients_only_cluster ├── README.md ├── compute.tf ├── datasources.tf ├── network.tf ├── outputs.tf ├── provider.tf └── variables.tf ├── clients_only_cluster_scripts ├── configure_nic.sh ├── create_spectrum_scale_cluster.sh ├── deploy_spectrum_scale.sh ├── firewall.sh ├── infra_tuning.sh ├── install_spectrum_scale.sh ├── nodes-cloud-init-complete-status-check.sh ├── passwordless_ssh.sh ├── set_env_variables.sh └── update_resolv_conf.sh ├── direct_attached ├── README.md ├── SharedDataBlockVolume.tf ├── compute.tf ├── datasources.tf ├── multi-attach-storage.tf ├── network.tf ├── outputs.tf ├── provider.tf ├── quorum.tf └── variables.tf ├── direct_attached_scripts ├── boot.sh ├── create_spectrum_scale_cluster.sh ├── deploy_spectrum_scale.sh ├── nodes-cloud-init-complete-status-check.sh └── nsddevices ├── images ├── 01-shared-nothing-architecture.png ├── 02-tf-apply.png ├── 03-mm-commands.png ├── 04-direct-attached-architecture.png ├── multi_cluster_spectrum_scale │ ├── 01_multi_cluster_spectrum_scale_architecture.png │ └── 02_multi_cluster_spectrum_scale_vcn_peering_architecture.png └── network_shared_disk_server_model │ ├── 01-high-level-architecture.png │ ├── 01-single-AD-architecture.png │ ├── 01-two-AD-architecture.png │ ├── 01a-single-AD-architecture.png │ ├── 01a-two-AD-architecture.png │ ├── 01b-single-AD-architecture.png │ ├── 02-tf-apply.png │ ├── 03-mm-commands.png │ ├── 04-gui-charts.png │ ├── 05-gui-dashboard.png │ ├── 06-gui-throughput.png │ ├── ss1.png │ ├── ss10.png │ ├── ss11_apply_output.png │ ├── ss2.png │ ├── ss3.png │ ├── ss4.png │ ├── ss5.png │ ├── ss6.png │ ├── ss7.png │ ├── ss8.png │ └── ss9.png ├── network_shared_disk_server_model.zip ├── network_shared_disk_server_model ├── README.md ├── ces.tf ├── compute.tf ├── compute_gpfs_management_gui.tf ├── datasources.tf ├── multi_attach_storage.tf ├── network.tf ├── outputs.tf ├── provider.tf ├── schema.yaml ├── scripts │ ├── block_volume_discovery.sh │ ├── configure_ces.sh │ ├── configure_mgmt_gui.sh │ ├── configure_nic.sh │ ├── create_spectrum_scale_cluster.sh │ ├── deploy_spectrum_scale.sh │ ├── firewall.sh │ ├── infra_tuning.sh │ ├── install_spectrum_scale.sh │ ├── nodes-cloud-init-complete-status-check.sh │ ├── nsddevices │ ├── passwordless_ssh.sh │ ├── set_env_variables.sh │ └── update_resolv_conf.sh ├── secondary_vnic.tf ├── shared_data_block_volume.tf ├── userdata │ ├── cloudinit.ps1 │ ├── cloudinit.yml │ └── setup.ps1 ├── variables.tf └── windows_smb_client.tf └── network_shared_disk_server_model_latest.zip /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Re-Store 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # oci-ibm-spectrum-scale 2 | These are Terraform modules that deploy IBM Spectrum Scale (GPFS) distributed parallel file system on [Oracle Cloud Infrastructure (OCI)](https://cloud.oracle.com/en_US/cloud-infrastructure). These were developed jointly by [Re-Store](https://www.re-store.net/) and Oracle Cloud team. 3 | 4 | On OCI, you can deploy Spectrum Scale using all the below architecture 5 | 6 | - Network Shared Disk (NSD) Server model - **Recommended** 7 | - Direct Attached Storage model 8 | - Shared Nothing model 9 | - Erasure Code Edition 10 | 11 | This templates uses the **Network Shared Disk (NSD) Server model** to deploy IBM Spectrum Scale. Please follow the instructions in [network_shared_disk_server_model](network_shared_disk_server_model) folders to deploy. To deploy other architecures, reach out to pinkesh.valdria@oracle.com . 12 | 13 | ## IBM Spectrum Scale 14 | IBM Spectrum Scale is a high-performance, highly available, clustered file system and associated management software, available on a variety of platforms. IBM Spectrum Scale can scale in several dimensions, including performance (bandwidth and IOPS), capacity, and number of nodes* (instances) that can mount the file system. IBM Spectrum Scale addresses the needs of applications whose performance (or performance-to-capacity ratio) demands cannot be met by traditional scale-up storage systems; and IBM Spectrum Scale is therefore deployed for many I/O-demanding enterprise applications that require high 15 | performance or scale. IBM Spectrum Scale provides various configuration options, access methods (including traditional POSIX-based file access), and many features such as snapshots, compression, and encryption. 16 | 17 | This Quick Start automates the deployment of IBM Spectrum Scale on OCI for users who require highly available access to a shared namespace across multiple instances with good performance, without requiring an in-depth knowledge of IBM Spectrum Scale. 18 | 19 | ## IBM Spectrum Scale Software Download 20 | This template is designed to work with the following editions: 21 | 22 | Spectrum Scale Data Management Edition 23 | Spectrum Scale Developer Edition - Free for upto 12TB Storage 24 | Spectrum Scale Data Access Edition 25 | 26 | 27 | Please download the Free developer edition of Spectrum Scale software binary from [IBM website](https://www.ibm.com/sg-en/marketplace/scale-out-file-and-object-storage/purchase). The software needs to be stored on a server which is accessible from the file servers created by this template in OCI. For example: you can save the software in private secure OCI Object Storage bucket and create pre-authenticated request to use in your template. 28 | 29 | If you already have license for Spectrum Scale, then you can download it from [here](https://www.ibm.com/support/fixcentral/swg/selectFixes?parent=Software%20defined%20storage&product=ibm/StorageSoftware/IBM+Spectrum+Scale&release=All&platform=Linux+64-bit,x86_64&function=all) 30 | 31 | 32 | ## Next Step 33 | 1. If your goal is to do a POC, we recommend deploying both Storage Server nodes and Client nodes together using instructions in **[network_shared_disk_server_model](network_shared_disk_server_model)** folders to deploy, including pre-requisites. 34 | 35 | 2. If your goal is to manage the Storage server nodes and Client nodes seperately (for production like setup), then follow the below steps 36 | - Create Storage Nodes only cluster using Terraform template in **[network_shared_disk_server_model](network_shared_disk_server_model)** folder. Make sure to set the client node count to zero. 37 | - Create "Clients only" cluster using Terraform template in **[clients_only_cluster](clients_only_cluster)** folder to provision client nodes to mount the filesystem. 38 | 39 | 3. If you have any questions, reach out to me at pinkesh.valdria@oracle.com 40 | -------------------------------------------------------------------------------- /clients_only_cluster/README.md: -------------------------------------------------------------------------------- 1 | # Spectrum Scale - Clients Only Cluster on OCI 2 | This Terrrafom template deploys a Spectrum Scale "clients only cluster". The "clients only cluster" will mount a remote Spectrum Scale File System cluster (Storage Cluster) to access data. Since its very common in HPC world to spin up/spin down a compute cluster, its best to manage client/compute nodes seperately from Spectrum Scale File System cluster (Storage Cluster). Also there could be multiple client clusters accessing a single Storage cluster. 3 | 4 | **Note:** This template assumes the Storage Spectrum Scale cluster was already created and this "clients_only_cluster" will not provision any server nodes or storage devices. For more information on how to create "Storage Spectrum Scale cluster", refer to this link: [Storage Spectrum Scale cluster](https://github.com/oracle-quickstart/oci-ibm-spectrum-scale/tree/master/network_shared_disk_server_model). 5 | 6 | 7 | ## Client Only Nodes Cluster architecture 8 | Given below are architecture diagram which show both the Clients Only Spectrum Scale cluster and Storage cluster, but this template will only provision/deploy the IaaS resources for "clients_only_cluster" and install/configure Spectrum Scale binaries. 9 | 10 | 11 | ### Multi Cluster Spectrum Scale 12 | ![](../images/multi_cluster_spectrum_scale/01_multi_cluster_spectrum_scale_architecture.png) 13 | 14 | ### Multi Cluster Spectrum Scale with VCN Peering 15 | ![](../images/multi_cluster_spectrum_scale/02_multi_cluster_spectrum_scale_vcn_peering_architecture.png) 16 | 17 | 18 | ## Prerequisites 19 | First off you'll need to do some pre deploy setup. That's all detailed [here](https://github.com/oracle/oci-quickstart-prerequisites). 20 | 21 | 22 | ## Clone the Terraform template 23 | Now, you'll want a local copy of this repo. You can make that with the commands: 24 | 25 | git clone https://github.com/oracle/oci-quickstart-ibm-spectrum-scale.git 26 | cd oci-quickstart-ibm-spectrum-scale/clients_only_cluster 27 | ls 28 | 29 | 30 | 31 | ## Update variables.tf file 32 | Update the variables.tf to set client compute shapes, # of client nodes, IBM Spectrum Scale binary download URL, version and various other values. 33 | 34 | 35 | - Update compute/client **node_count** and **shape** to provision. A minimum of 3 client nodes are required to maintain quorum in a cluster. There are ways to overcome this requirement for production, but that's not covered in this deployment template. 36 | 37 | ``` 38 | Client nodes variables. Min node required is 3 to have quorum. For production, with some customization, less than 3 nodes can be supported. 39 | variable "client_node" { 40 | type = map(string) 41 | default = { 42 | shape = "VM.Standard2.4" 43 | node_count = 3 44 | hostname_prefix = "ss-compute-" 45 | } 46 | } 47 | ``` 48 | 49 | - Update the **version** and **download_url** variables based on version of Spectrum Scale you plan to deploy. We recommend you upload the Spectrum Scale binary file to OCI Object Storage bucket (private) and create a pre-authenticated URL to access it from all the spectrum scale nodes to be provisioned by the template. You can also use any other server to host the binaries, provided the server is reachable over the internet without any authentication (no username/password, etc). 50 | 51 | - **download_url** : Should be a http/https link which is accessible from all Spectrum Scale instances we will create. Assuming you have uploaded the spectrum scale software binary to OCI Object storage private bucket. You can create a preauthenticatedrequests using the steps detailed here - [create pre-authenticated url](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/usingpreauthenticatedrequests.htm#usingconsole) 52 | 53 | - **Note:** The name of the spectrum scale software binary file needs to exactly follow this naming convention. These are the names of the file by default, when you download it from IBM site. 54 | - For Spectrum Scale Data Management Edition: Spectrum_Scale_Data_Management-5.0.3.3-x86_64-Linux-install 55 | - For Spectrum Scale Developer Edition: Spectrum Scale 5.0.4.1 Developer Edition.zip 56 | - Once you upload to OCI Object Storage, the download_url will look like one of the below: 57 | - https://objectstorage.us-ashburn-1.oraclecloud.com/xxxxxxxx/Spectrum_Scale_Data_Management-5.0.3.3-x86_64-Linux-install 58 | - https://objectstorage.us-ashburn-1.oraclecloud.com/xxxxxxxx/Spectrum%20Scale%205.0.4.1%20Developer%20Edition.zip 59 | 60 | - **version** : Replace the **version** number with the one you plan to use, eg: 5.0.4.1 61 | 62 | 63 | 64 | 65 | - Is the client only cluster going to be deployed in an existing VCN? 66 | - If no, set **use_existing_vcn** to false, and ignore the **vcn_id, bastion_subnet_id & fs_subnet_id** variable. The template will create a new VCN using 172.16.0.0/16 CIDR. Make sure this VCN CIDR does not overlap with VCN CIDR of "Spectrum Scale Storage" cluster. If it overlaps, then update the **vpc_cidr** variable to use a different non-overlapping CIDR. 67 | 68 | - If yes, set the following variables: 69 | 70 | 71 | - Set **use_existing_vcn** to true 72 | - Set **vcn_id** to existing VCN OCID 73 | - Set **bastion_subnet_id** to existing public subnet OCID of existing VCN 74 | - Set **fs_subnet_id** to existing private subnet OCID of an existing VCN based on below conditions. 75 | 76 | 77 | 78 | - If the "Spectrum Scale Storage" cluster is also deployed in the same existing VCN, then check 79 | If Spectrum Scale servers are Baremetal nodes, then this should be the "Private-FS-Subnet" subnet OCID. If Spectrum Scale servers are VMs or BM.HPC2.36, then this should be the "Private-SpectrumScale" subnet OCID. Refer to VCN/Subnet details of "Server Spectrum Scale" Cluster. 80 | fs_subnet_id 81 | 82 | - If the "Spectrum Scale Storage" cluster is **NOT** deployed in the same existing VCN and you plan to use VCN local/remote peering, then provide a private subnet OCID you plan to use to provision the client/compute nodes. 83 | 84 | 85 | 86 | ## Deployment 87 | 88 | Deploy using standard Terraform commands. 89 | 90 | 91 | cd oci-quickstart-ibm-spectrum-scale/clients_only_cluster 92 | terraform init && terraform plan 93 | terraform apply 94 | 95 | 96 | ## Post Deploy Steps 97 | Once the "clients_only_cluster" is deployed, there are some post deploy steps to establish authentication between the "clients_ clusters and to get the filesystem mounted on client/compute nodes. The detailed steps are documented [here](https://www.ibm.com/support/knowledgecenter/STXKQY_5.0.5/com.ibm.spectrum.scale.v5r05.doc/bl1adv_admrmsec.htm). 98 | 99 | 100 | -------------------------------------------------------------------------------- /clients_only_cluster/compute.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "oci_core_instance" "client_node" { 3 | count = var.client_node["node_count"] 4 | availability_domain = lookup(data.oci_identity_availability_domains.ADs.availability_domains[( (count.index < (var.client_node["node_count"] / 2)) ? local.site1 : local.site2)],"name") 5 | 6 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 7 | compartment_id = var.compartment_ocid 8 | display_name = "${var.client_node["hostname_prefix"]}${format("%01d", count.index+1)}" 9 | hostname_label = "${var.client_node["hostname_prefix"]}${format("%01d", count.index+1)}" 10 | shape = var.client_node["shape"] 11 | subnet_id = local.client_subnet_id 12 | 13 | source_details { 14 | source_type = "image" 15 | source_id = var.images[var.region] 16 | } 17 | 18 | launch_options { 19 | network_type = (length(regexall("M.Standard.E", var.client_node["shape"])) > 0 ? "PARAVIRTUALIZED" : "VFIO") 20 | } 21 | 22 | metadata = { 23 | ssh_authorized_keys = var.ssh_public_key 24 | user_data = "${base64encode(join("\n", list( 25 | "#!/usr/bin/env bash", 26 | "set -x", 27 | "version=\"${var.spectrum_scale["version"]}\"", 28 | "downloadUrl=\"${var.spectrum_scale["download_url"]}\"", 29 | "sshPrivateKey=\"${var.ssh_private_key}\"", 30 | "sshPublicKey=\"${var.ssh_public_key}\"", 31 | "clientNodeCount=\"${var.client_node["node_count"]}\"", 32 | "clientNodeHostnamePrefix=\"${var.client_node["hostname_prefix"]}\"", 33 | "installerNode=\"${var.client_node["hostname_prefix"]}${var.installer_node}\"", 34 | "vcnFQDN=\"${local.vcn_domain_name}\"", 35 | "privateBSubnetsFQDN=\"${local.filesystem_subnet_domain_name}\"", 36 | file("${var.scripts_directory}/firewall.sh"), 37 | file("${var.scripts_directory}/set_env_variables.sh"), 38 | file("${var.scripts_directory}/update_resolv_conf.sh"), 39 | file("${var.scripts_directory}/configure_nic.sh"), 40 | file("${var.scripts_directory}/infra_tuning.sh"), 41 | file("${var.scripts_directory}/passwordless_ssh.sh"), 42 | file("${var.scripts_directory}/install_spectrum_scale.sh") 43 | )))}" 44 | } 45 | 46 | timeouts { 47 | create = "120m" 48 | } 49 | 50 | } 51 | 52 | 53 | 54 | /* bastion instances */ 55 | resource "oci_core_instance" "bastion" { 56 | count = var.bastion["node_count"] 57 | availability_domain = lookup(data.oci_identity_availability_domains.ADs.availability_domains[((count.index % 2 == 0) ? local.site1 : local.site2)],"name") 58 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 59 | compartment_id = var.compartment_ocid 60 | display_name = "${var.bastion["hostname_prefix"]}${format("%01d", count.index+1)}" 61 | shape = var.bastion["shape"] 62 | hostname_label = "${var.bastion["hostname_prefix"]}${format("%01d", count.index+1)}" 63 | 64 | create_vnic_details { 65 | subnet_id = local.bastion_subnet_id 66 | #"ocid1.instance.oc1.iad.anuwcljtpwneysacxazpgiz2gdej7tzvpzkjycirlqbweowk7624sllkgluq" 67 | #1# subnet_id = oci_core_subnet.public.*.id[0] 68 | skip_source_dest_check = true 69 | } 70 | 71 | metadata = { 72 | ssh_authorized_keys = var.ssh_public_key 73 | } 74 | 75 | source_details { 76 | source_type = "image" 77 | source_id = var.images[var.region] 78 | } 79 | } 80 | 81 | 82 | 83 | /* Remote exec to deploy gpfs software/rpms on client nodes */ 84 | resource "null_resource" "deploy_gpfs_on_client_nodes" { 85 | depends_on = [ 86 | oci_core_instance.client_node 87 | ] 88 | count = var.client_node["node_count"] 89 | triggers = { 90 | instance_ids = "oci_core_instance.client_node.*.id" 91 | } 92 | 93 | provisioner "file" { 94 | source = "${var.scripts_directory}/" 95 | destination = "/tmp/" 96 | connection { 97 | agent = false 98 | timeout = "30m" 99 | host = element(oci_core_instance.client_node.*.private_ip, count.index) 100 | user = var.ssh_user 101 | private_key = var.ssh_private_key 102 | bastion_host = oci_core_instance.bastion[0].public_ip 103 | bastion_port = "22" 104 | bastion_user = var.ssh_user 105 | bastion_private_key = var.ssh_private_key 106 | } 107 | } 108 | 109 | provisioner "remote-exec" { 110 | connection { 111 | agent = false 112 | timeout = "30m" 113 | host = element(oci_core_instance.client_node.*.private_ip, count.index) 114 | user = var.ssh_user 115 | private_key = var.ssh_private_key 116 | bastion_host = oci_core_instance.bastion[0].public_ip 117 | bastion_port = "22" 118 | bastion_user = var.ssh_user 119 | bastion_private_key = var.ssh_private_key 120 | } 121 | inline = [ 122 | "set -x", 123 | "echo about to run /tmp/nodes-cloud-init-complete-status-check.sh", 124 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 125 | "sudo -s bash -c 'set -x && /tmp/nodes-cloud-init-complete-status-check.sh'", 126 | "sudo -s bash -c 'set -x && /tmp/deploy_spectrum_scale.sh'", 127 | ] 128 | } 129 | } 130 | 131 | 132 | 133 | 134 | /* Remote exec to create gpfs cluster on installer node */ 135 | resource "null_resource" "create_gpfs_cluster" { 136 | depends_on = [ 137 | oci_core_instance.client_node, 138 | null_resource.deploy_gpfs_on_client_nodes 139 | ] 140 | count = 1 141 | triggers = { 142 | instance_ids = "oci_core_instance.client_node.*.id" 143 | } 144 | 145 | provisioner "remote-exec" { 146 | connection { 147 | agent = false 148 | timeout = "30m" 149 | host = element(oci_core_instance.client_node.*.private_ip, count.index) 150 | user = var.ssh_user 151 | private_key = var.ssh_private_key 152 | bastion_host = oci_core_instance.bastion[0].public_ip 153 | bastion_port = "22" 154 | bastion_user = var.ssh_user 155 | bastion_private_key = var.ssh_private_key 156 | } 157 | inline = [ 158 | "set -x", 159 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 160 | "sudo su -l -c 'set -x && /tmp/create_spectrum_scale_cluster.sh'", 161 | ] 162 | } 163 | } 164 | 165 | 166 | 167 | 168 | -------------------------------------------------------------------------------- /clients_only_cluster/datasources.tf: -------------------------------------------------------------------------------- 1 | # Gets a list of Availability Domains 2 | data "oci_identity_availability_domains" "ADs" { 3 | compartment_id = "${var.tenancy_ocid}" 4 | } 5 | 6 | data "oci_core_vcn" "vcn" { 7 | vcn_id = var.use_existing_vcn ? var.vcn_id : oci_core_vcn.vcn[0].id 8 | } 9 | 10 | data "oci_core_subnet" "fs_subnet" { 11 | subnet_id = var.use_existing_vcn ? var.fs_subnet_id : local.fs_subnet_id 12 | } 13 | -------------------------------------------------------------------------------- /clients_only_cluster/network.tf: -------------------------------------------------------------------------------- 1 | /* 2 | All network resources for this template 3 | */ 4 | 5 | resource "oci_core_vcn" "vcn" { 6 | count = var.use_existing_vcn ? 0 : 1 7 | cidr_block = var.vpc_cidr 8 | compartment_id = var.compartment_ocid 9 | display_name = "gpfs" 10 | dns_label = "gpfs" 11 | } 12 | 13 | resource "oci_core_internet_gateway" "internet_gateway" { 14 | count = var.use_existing_vcn ? 0 : 1 15 | compartment_id = var.compartment_ocid 16 | display_name = "internet_gateway" 17 | vcn_id = oci_core_vcn.vcn[0].id 18 | } 19 | 20 | resource "oci_core_route_table" "pubic_route_table" { 21 | count = var.use_existing_vcn ? 0 : 1 22 | compartment_id = var.compartment_ocid 23 | vcn_id = oci_core_vcn.vcn[0].id 24 | display_name = "RouteTableForComplete" 25 | route_rules { 26 | cidr_block = "0.0.0.0/0" 27 | network_entity_id = oci_core_internet_gateway.internet_gateway[0].id 28 | } 29 | } 30 | 31 | 32 | resource "oci_core_nat_gateway" "nat_gateway" { 33 | count = var.use_existing_vcn ? 0 : 1 34 | compartment_id = var.compartment_ocid 35 | vcn_id = oci_core_vcn.vcn[0].id 36 | display_name = "nat_gateway" 37 | } 38 | 39 | 40 | resource "oci_core_route_table" "private_route_table" { 41 | count = var.use_existing_vcn ? 0 : 1 42 | compartment_id = var.compartment_ocid 43 | vcn_id = oci_core_vcn.vcn[0].id 44 | display_name = "private_route_tableForComplete" 45 | route_rules { 46 | destination = "0.0.0.0/0" 47 | network_entity_id = oci_core_nat_gateway.nat_gateway[0].id 48 | } 49 | } 50 | 51 | resource "oci_core_security_list" "public_security_list" { 52 | count = var.use_existing_vcn ? 0 : 1 53 | compartment_id = var.compartment_ocid 54 | display_name = "Public Subnet" 55 | vcn_id = oci_core_vcn.vcn[0].id 56 | egress_security_rules { 57 | destination = "0.0.0.0/0" 58 | protocol = "6" 59 | } 60 | ingress_security_rules { 61 | tcp_options { 62 | max = 22 63 | min = 22 64 | } 65 | protocol = "6" 66 | source = "0.0.0.0/0" 67 | } 68 | ingress_security_rules { 69 | tcp_options { 70 | max = 3389 71 | min = 3389 72 | } 73 | protocol = "6" 74 | source = "0.0.0.0/0" 75 | } 76 | } 77 | 78 | # https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewall.htm 79 | resource "oci_core_security_list" "private_security_list" { 80 | count = var.use_existing_vcn ? 0 : 1 81 | compartment_id = var.compartment_ocid 82 | display_name = "Private" 83 | vcn_id = oci_core_vcn.vcn[0].id 84 | 85 | egress_security_rules { 86 | destination = "0.0.0.0/0" 87 | protocol = "all" 88 | } 89 | egress_security_rules { 90 | protocol = "all" 91 | destination = var.vpc_cidr 92 | } 93 | ingress_security_rules { 94 | tcp_options { 95 | max = 22 96 | min = 22 97 | } 98 | protocol = "6" 99 | source = var.vpc_cidr 100 | } 101 | 102 | ingress_security_rules { 103 | protocol = "All" 104 | source = var.vpc_cidr 105 | } 106 | } 107 | 108 | 109 | # Regional subnet - public 110 | resource "oci_core_subnet" "public" { 111 | count = var.use_existing_vcn ? 0 : 1 112 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index) 113 | #display_name = "${local.cluster_name}_public" 114 | display_name = "Public-Subnet" 115 | compartment_id = var.compartment_ocid 116 | vcn_id = oci_core_vcn.vcn[0].id 117 | route_table_id = oci_core_route_table.pubic_route_table[0].id 118 | security_list_ids = [oci_core_security_list.public_security_list[0].id] 119 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 120 | dns_label = "public" 121 | } 122 | 123 | # Regional subnet - private 124 | resource "oci_core_subnet" "storage" { 125 | count = var.use_existing_vcn ? 0 : 1 126 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index+3) 127 | display_name = "Private-SpectrumScale" 128 | compartment_id = var.compartment_ocid 129 | vcn_id = oci_core_vcn.vcn[0].id 130 | route_table_id = oci_core_route_table.private_route_table[0].id 131 | security_list_ids = [oci_core_security_list.private_security_list[0].id] 132 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 133 | prohibit_public_ip_on_vnic = "true" 134 | dns_label = "storage" 135 | } 136 | 137 | 138 | # Regional subnet - private B 139 | resource "oci_core_subnet" "fs" { 140 | count = var.use_existing_vcn ? 0 : 1 141 | #(local.dual_nics ? 1 : 0) 142 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index+6) 143 | display_name = "Private-FS-Subnet" 144 | compartment_id = var.compartment_ocid 145 | vcn_id = oci_core_vcn.vcn[0].id 146 | route_table_id = oci_core_route_table.private_route_table[0].id 147 | security_list_ids = [oci_core_security_list.private_security_list[0].id] 148 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 149 | prohibit_public_ip_on_vnic = "true" 150 | dns_label = "fs" 151 | } 152 | 153 | 154 | 155 | 156 | -------------------------------------------------------------------------------- /clients_only_cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | output "SSH-login" { 4 | value = < 0 ? true : false) 86 | bastion_subnet_id = var.use_existing_vcn ? var.bastion_subnet_id : element(concat(oci_core_subnet.public.*.id, [""]), 0) 87 | fs_subnet_id = var.use_existing_vcn ? var.fs_subnet_id : element(concat(oci_core_subnet.fs.*.id, [""]), 0) 88 | client_subnet_id = local.fs_subnet_id 89 | filesystem_subnet_domain_name= ("${data.oci_core_subnet.fs_subnet.dns_label}.${data.oci_core_vcn.vcn.dns_label}.oraclevcn.com" ) 90 | vcn_domain_name=("${data.oci_core_vcn.vcn.dns_label}.oraclevcn.com" ) 91 | 92 | } 93 | 94 | # Please do not change. The first nsd node is used for cluster deployment 95 | variable "installer_node" { default = "1" } 96 | 97 | variable "scripts_directory" { default = "../clients_only_cluster_scripts" } 98 | 99 | variable "tenancy_ocid" {} 100 | variable "user_ocid" {} 101 | variable "fingerprint" {} 102 | variable "private_key_path" {} 103 | #variable "region" { default = "us-ashburn-1" } 104 | variable "region" { default = "uk-london-1" } 105 | 106 | variable "compartment_ocid" {} 107 | variable "ssh_public_key" {} 108 | variable "ssh_private_key" {} 109 | variable "ssh_private_key_path" {} 110 | 111 | /* 112 | For instances created using Oracle Linux and CentOS images, the user name opc is created automatically. 113 | For instances created using the Ubuntu image, the user name ubuntu is created automatically. 114 | The ubuntu user has sudo privileges and is configured for remote access over the SSH v2 protocol using RSA keys. The SSH public keys that you specify while creating instances are added to the /home/ubuntu/.ssh/authorized_keys file. 115 | For more details: https://docs.cloud.oracle.com/iaas/Content/Compute/References/images.htm#one 116 | For Ubuntu images, set to ubuntu. 117 | # variable "ssh_user" { default = "ubuntu" } 118 | */ 119 | variable "ssh_user" { default = "opc" } 120 | 121 | 122 | 123 | /* 124 | See https://docs.us-phoenix-1.oraclecloud.com/images/ or https://docs.cloud.oracle.com/iaas/images/ 125 | Oracle-provided image "CentOS-7-2019.08.26-0" 126 | https://docs.cloud.oracle.com/iaas/images/image/ea67dd20-b247-4937-bfff-894962212415/ 127 | */ 128 | /* imagesCentOS */ 129 | variable "images" { 130 | type = map(string) 131 | default = { 132 | ap-mumbai-1 = "ocid1.image.oc1.ap-mumbai-1.aaaaaaaabfqn5vmh3pg6ynpo6bqdbg7fwruu7qgbvondjic5ccr4atlj4j7q" 133 | ap-seoul-1 = "ocid1.image.oc1.ap-seoul-1.aaaaaaaaxfeztdrbpn452jk2yln7imo4leuhlqicoovoqu7cxqhkr3j2zuqa" 134 | ap-sydney-1 = "ocid1.image.oc1.ap-sydney-1.aaaaaaaanrubykp6xrff5xzd6gu2g6ul6ttnyoxgaeeq434urjz5j6wfq4fa" 135 | ap-tokyo-1 = "ocid1.image.oc1.ap-tokyo-1.aaaaaaaakkqtoabcjigninsyalinvppokmgaza6amynam3gs2ldelpgesu6q" 136 | ca-toronto-1 = "ocid1.image.oc1.ca-toronto-1.aaaaaaaab4hxrwlcs4tniwjr4wvqocmc7bcn3apnaapxabyg62m2ynwrpe2a" 137 | eu-frankfurt-1 = "ocid1.image.oc1.eu-frankfurt-1.aaaaaaaawejnjwwnzapqukqudpczm4pwtpcsjhohl7qcqa5vzd3gxwmqiq3q" 138 | eu-zurich-1 = "ocid1.image.oc1.eu-zurich-1.aaaaaaaa7hdfqf54qcnu3bizufapscopzdlxp54yztuxauxyraprxnqjj7ia" 139 | sa-saopaulo-1 = "ocid1.image.oc1.sa-saopaulo-1.aaaaaaaa2iqobvkeowx4n2nqsgy32etohkw2srqireqqk3bhn6hv5275my6a" 140 | uk-london-1 = "ocid1.image.oc1.uk-london-1.aaaaaaaakgrjgpq3jej3tyqfwsyk76tl25zoflqfjjuuv43mgisrmhfniofq" 141 | us-ashburn-1 = "ocid1.image.oc1.iad.aaaaaaaa5phjudcfeyomogjp6jjtpcl3ozgrz6s62ltrqsfunejoj7cqxqwq" 142 | us-phoenix-1 = "ocid1.image.oc1.phx.aaaaaaaag7vycom7jhxqxfl6rxt5pnf5wqolksl6onuqxderkqrgy4gsi3hq" 143 | } 144 | } 145 | 146 | variable "imagesCentos76" { 147 | type = map(string) 148 | default = { 149 | /* 150 | See https://docs.us-phoenix-1.oraclecloud.com/images/ or https://docs.cloud.oracle.com/iaas/images/ 151 | Oracle-provided image "CentOS-7-2018.11.16-0" 152 | https://docs.cloud.oracle.com/iaas/images/image/66a17669-8a67-4b43-924a-78d8ae49f609/ 153 | */ 154 | eu-frankfurt-1 = "ocid1.image.oc1.eu-frankfurt-1.aaaaaaaatbfzohfzwagb5eplk5abjifwmr5bpytuo2pgyufflpkdfkkb3eca" 155 | us-ashburn-1 = "ocid1.image.oc1.iad.aaaaaaaa3p2d4bzgz4gw435tw3522u4d3enh7jwlwpymfgqwp6hrhebs4s2q" 156 | uk-london-1 = "ocid1.image.oc1.uk-london-1.aaaaaaaaktvxlhhjs3k57fbloubrbuju7vdyaivdw5pclmva2kwhqhqlewbq" 157 | us-phoenix-1 = "ocid1.image.oc1.phx.aaaaaaaavzt7r56xh2lj2w7ibqbkvumxbqr2z2jswoma3qjbunu7wj63rigq" 158 | } 159 | } 160 | 161 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/configure_nic.sh: -------------------------------------------------------------------------------- 1 | MDATA_VNIC_URL="http://169.254.169.254/opc/v1/vnics/" 2 | 3 | function configure_vnics { 4 | # Configure second vNIC 5 | scriptsource="https://raw.githubusercontent.com/oracle/terraform-examples/master/examples/oci/connect_vcns_using_multiple_vnics/scripts/secondary_vnic_all_configure.sh" 6 | vnicscript=/root/secondary_vnic_all_configure.sh 7 | curl -s $scriptsource > $vnicscript 8 | chmod +x $vnicscript 9 | cat > /etc/systemd/system/secondnic.service << EOF 10 | [Unit] 11 | Description=Script to configure a secondary vNIC 12 | 13 | [Service] 14 | Type=oneshot 15 | ExecStart=$vnicscript -c 16 | ExecStop=$vnicscript -d 17 | RemainAfterExit=yes 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | 22 | EOF 23 | 24 | systemctl enable secondnic.service 25 | systemctl start secondnic.service 26 | sleep 10s 27 | vnic_cnt=`/root/secondary_vnic_all_configure.sh | grep "ocid1.vnic." | grep " UP " | wc -l` ; 28 | RC=1 29 | interface="" 30 | while ( [ $vnic_cnt -le 1 ] || [ $RC -ne 0 ] ) 31 | do 32 | systemctl restart secondnic.service 33 | echo "sleep 10s" 34 | sleep 10s 35 | privateIp=`curl -s http://169.254.169.254/opc/v1/vnics/ | jq '.[1].privateIp ' | sed 's/"//g' ` ; echo $privateIp 36 | interface=`ip addr | grep -B2 $privateIp | grep "BROADCAST" | gawk -F ":" ' { print $2 } ' | sed -e 's/^[ \t]*//'` ; echo $interface 37 | if [ -z $interface ]; then 38 | # repeat loop 39 | RC=1 40 | else 41 | RC=0 42 | fi 43 | vnic_cnt=`/root/secondary_vnic_all_configure.sh | grep "ocid1.vnic." | grep " UP " | wc -l` ; 44 | done 45 | 46 | } 47 | 48 | function configure_2nd_VNIC { 49 | 50 | configure_vnics 51 | # check if 1 or 2 VNIC. 52 | vnic_count=`curl -s $MDATA_VNIC_URL | jq '. | length'` 53 | 54 | if [ $vnic_count -gt 1 ] ; then 55 | echo "2 VNIC setup" 56 | 57 | privateIp=`curl -s $MDATA_VNIC_URL | jq '.[1].privateIp ' | sed 's/"//g' ` ; echo $privateIp 58 | interface=`ip addr | grep -B2 $privateIp | grep "BROADCAST" | gawk -F ":" ' { print $2 } ' | sed -e 's/^[ \t]*//'` ; echo $interface 59 | 60 | if [ "$intel_node" = "true" ]; then 61 | if [ "$hpc_node" = "true" ]; then 62 | echo "don't tune on hpc shape" 63 | else 64 | tune_interface 65 | fi 66 | fi 67 | 68 | # ensure the below is not empty 69 | test=`nslookup $privateIp | grep -q "name = "` 70 | while [ $? -ne 0 ]; 71 | do 72 | echo "Waiting for nslookup..." 73 | sleep 10s 74 | test=`nslookup $privateIp | grep -q "name = "` 75 | done 76 | 77 | secondNicFQDNHostname=`nslookup $privateIp | grep "name = " | gawk -F"=" '{ print $2 }' | sed "s|^ ||g" | sed "s|\.$||g"` 78 | thisFQDN=$secondNicFQDNHostname 79 | thisHost=${thisFQDN%%.*} 80 | secondNICDomainName=${thisFQDN#*.*} 81 | echo $secondNICDomainName 82 | primaryNICHostname="`hostname`" 83 | else 84 | echo "Server nodes with 1 physical NIC - get hostname for 1st NIC..." 85 | set_env_variables 86 | fi 87 | } 88 | 89 | function tune_interface { 90 | ethtool -G $interface rx 2047 tx 2047 rx-jumbo 8191 91 | ethtool -L $interface combined 74 92 | echo "ethtool -G $interface rx 2047 tx 2047 rx-jumbo 8191" >> /etc/rc.local 93 | echo "ethtool -L $interface combined 74" >> /etc/rc.local 94 | chmod +x /etc/rc.local 95 | } 96 | 97 | function set_env_variables { 98 | thisFQDN=`hostname --fqdn` 99 | thisHost=${thisFQDN%%.*} 100 | } 101 | 102 | coreIdCount=`grep "^core id" /proc/cpuinfo | sort -u | wc -l` ; 103 | socketCount=`echo $(($(grep "^physical id" /proc/cpuinfo | awk '{print $4}' | sort -un | tail -1)+1))` ; 104 | 105 | 106 | # configure 1st NIC 107 | privateIp=`curl -s $MDATA_VNIC_URL | jq '.[0].privateIp ' | sed 's/"//g' ` ; echo $privateIp 108 | interface=`ip addr | grep -B2 $privateIp | grep "BROADCAST" | gawk -F ":" ' { print $2 } ' | sed -e 's/^[ \t]*//'` ; echo $interface 109 | 110 | hpc_node=false 111 | intel_node=true 112 | lscpu | grep "Vendor ID:" | grep "AuthenticAMD" 113 | if [ $? -eq 0 ]; then 114 | echo "do nothing - AMD" 115 | intel_node=false 116 | else 117 | if [ $((socketCount*coreIdCount)) -eq 36 ]; then 118 | echo "skip for hpc" 119 | hpc_node=true 120 | else 121 | tune_interface 122 | fi 123 | fi 124 | 125 | set_env_variables 126 | # required to include in this file 127 | echo "thisFQDN=\"$thisFQDN\"" >> /tmp/gpfs_env_variables.sh 128 | echo "thisHost=\"$thisHost\"" >> /tmp/gpfs_env_variables.sh 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/create_spectrum_scale_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | source /tmp/gpfs_env_variables.sh 6 | 7 | 8 | # Only on installerNode 9 | echo "$thisHost" | grep -q -w $installerNode 10 | if [ $? -eq 0 ] ; then 11 | rm /root/node.stanza 12 | count=0 13 | for hname in `cat /tmp/allnodehosts` ; do 14 | if [ $count -lt 3 ] ; then 15 | echo "${hname}:quorum" >> /root/node.stanza 16 | else 17 | echo "${hname}" >> /root/node.stanza 18 | fi 19 | count=$((count+1)) 20 | done 21 | cat /root/node.stanza 22 | 23 | mmcrcluster -N node.stanza -r /usr/bin/ssh -R /usr/bin/scp -C ss-compute-only -A 24 | sleep 30s 25 | 26 | count=0 27 | for hname in `cat /tmp/allnodehosts` ; do 28 | if [ $count -lt 3 ] ; then 29 | mmchlicense server --accept -N ${hname} 30 | else 31 | mmchlicense client --accept -N ${hname} 32 | fi 33 | count=$((count+1)) 34 | done 35 | 36 | sleep 30s 37 | mmstartup -a 38 | while [ `mmgetstate -a | grep "active" | wc -l` -lt $((clientNodeCount)) ] ; do echo "waiting for client nodes of cluster to start ..." ; sleep 10s; done; 39 | 40 | mmlscluster 41 | mmlsnodeclass --all 42 | 43 | # https://www.ibm.com/support/knowledgecenter/STXKQY_5.0.5/com.ibm.spectrum.scale.v5r05.doc/bl1adv_admrmsec.htm 44 | mmauth genkey new 45 | mmauth update . -l AUTHONLY 46 | cp /var/mmfs/ssl/id_rsa.pub /home/opc/accessingCluster_id_rsa.pub 47 | chown opc:opc /home/opc/accessingCluster_id_rsa.pub 48 | 49 | # run the below on ss-compute-1 node, after the ss-compute-only cluster is provisioned and you have the value of accessingClusterName and accessingCluster_id_rsa.pub file copied to ss-server-1 server. example: 50 | # owningClusterName=ss-storage-cluster.storage.gpfs.oraclevcn.com 51 | # owningClusterAuthPublicKeyFilePath=/home/opc/owningCluster_id_rsa.pub 52 | # contactNodes are all NSD nodes. node1,node2,node3 53 | # owningClusterContactNodes=ss-server-1.storage.gpfs.oraclevcn.com,ss-server-2.storage.gpfs.oraclevcn.com 54 | # filesystemName - name of the filesystem, eg: fs1 is the default in server only cluster. 55 | # filesystemName=fs1 56 | 57 | ##mmremotecluster add $owningClusterName -n $owningClusterContactNodes -k $owningClusterAuthPublicKeyFilePath 58 | 59 | # Only run these after this cluster (accessingCluster) information is configured on the owningCluster. 60 | # mmremotefs add remote${filesystemName} -f ${filesystemName} -C $owningClusterName -T /remote${filesystemName} 61 | # It needs to be manually mounted on each node 62 | # filesystemName=fs1 ; mmmount remote${filesystemName} 63 | 64 | 65 | fi 66 | 67 | exit 0; 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/deploy_spectrum_scale.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | source /tmp/gpfs_env_variables.sh 6 | 7 | 8 | # Build the GPFS potability layer. 9 | /usr/lpp/mmfs/bin/mmbuildgpl 10 | # have seen the above fail sometimes, hence the below loop 11 | while [ $? -ne 0 ]; do 12 | sleep 10s; 13 | /usr/lpp/mmfs/bin/mmbuildgpl 14 | done; 15 | 16 | 17 | # Up date the PATH environmental variable. 18 | echo -e '\nexport PATH=/usr/lpp/mmfs/bin:$PATH' >> ~/.bash_profile 19 | source ~/.bash_profile 20 | 21 | exit 0; 22 | 23 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/firewall.sh: -------------------------------------------------------------------------------- 1 | echo "Turning off the Firewall..." 2 | service firewalld stop 3 | chkconfig firewalld off 4 | 5 | echo `hostname` | grep -q "$clientNodeHostnamePrefix" 6 | if [ $? -eq 0 ] ; then 7 | echo "continue, no sleep required" 8 | else 9 | coreIdCount=`grep "^core id" /proc/cpuinfo | sort -u | wc -l` 10 | socketCount=`echo $(($(grep "^physical id" /proc/cpuinfo | awk '{print $4}' | sort -un | tail -1)+1))` 11 | if [ $((socketCount*coreIdCount)) -eq 36 ]; then 12 | sleep 900s 13 | fi 14 | fi 15 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/infra_tuning.sh: -------------------------------------------------------------------------------- 1 | mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.disabled 2 | mv /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel-testing.repo.disabled 3 | sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config 4 | setenforce 0 5 | 6 | ### OS Performance tuning 7 | cd /usr/lib/tuned/ 8 | cp -r throughput-performance/ gpfs-oci-performance 9 | 10 | echo " 11 | # 12 | # tuned configuration 13 | # 14 | 15 | [main] 16 | summary=gpfs perf tuning for common gpfs workloads 17 | 18 | [cpu] 19 | force_latency=1 20 | governor=performance 21 | energy_perf_bias=performance 22 | min_perf_pct=100 23 | 24 | [vm] 25 | transparent_huge_pages=never 26 | 27 | [sysctl] 28 | net.ipv4.tcp_timestamps=1 29 | net.ipv4.tcp_sack=1 30 | net.ipv4.tcp_dsack=1 31 | net.ipv4.tcp_low_latency=1 32 | net.ipv4.tcp_adv_win_scale=2 33 | net.ipv4.tcp_window_scaling=1 34 | net.ipv4.tcp_slow_start_after_idle=0 35 | net.ipv4.tcp_syn_retries=8 36 | net.ipv4.tcp_rmem=4096 87380 16777216 37 | net.ipv4.tcp_wmem=4096 65536 16777216 38 | net.core.rmem_max=16777216 39 | net.core.wmem_max=16777216 40 | net.core.rmem_default=16777216 41 | net.core.wmem_default=16777216 42 | net.core.optmem_max=16777216 43 | net.core.somaxconn = 8192 44 | net.core.netdev_max_backlog=250000 45 | sunrpc.udp_slot_table_entries=128 46 | sunrpc.tcp_slot_table_entries=128 47 | kernel.sysrq = 1 48 | kernel.sched_min_granularity_ns = 10000000 49 | kernel.sched_wakeup_granularity_ns = 15000000 50 | vm.min_free_kbytes = 16777216 51 | vm.dirty_ratio = 30 52 | vm.dirty_background_ratio = 10 53 | vm.swappiness=30 54 | " > gpfs-oci-performance/tuned.conf 55 | 56 | cd - 57 | 58 | 59 | # make sure client they have enough memory. 60 | echo "$thisHost" | grep -q $clientNodeHostnamePrefix 61 | if [ $? -eq 0 ] ; then 62 | coreIdCount=`grep "^core id" /proc/cpuinfo | sort -u | wc -l` ; echo $coreIdCount 63 | socketCount=`echo $(($(grep "^physical id" /proc/cpuinfo | awk '{print $4}' | sort -un | tail -1)+1))` ; echo $socketCount 64 | if [ $((socketCount*coreIdCount)) -gt 4 ]; then 65 | tuned-adm profile gpfs-oci-performance 66 | else 67 | # Client is using shape with less than 4 physical cores and less 30GB memory, above tuned profile requires atleast 16GB of vm.min_free_kbytes, hence let user evaluate what are valid values for such small compute shapes. 68 | echo "skip profile tuning..." 69 | fi ; 70 | fi; 71 | 72 | tuned-adm active 73 | 74 | echo "$thisHost" | grep -q $clientNodeHostnamePrefix 75 | if [ $? -eq 0 ] ; then 76 | echo off > /sys/devices/system/cpu/smt/control 77 | fi 78 | 79 | echo "$thisHost" | grep -q $clientNodeHostnamePrefix 80 | if [ $? -eq 0 ] ; then 81 | 82 | echo ' 83 | * soft memlock -1 84 | * hard memlock -1 85 | * soft rss -1 86 | * hard rss -1 87 | * soft core -1 88 | * hard core -1 89 | * soft maxlogins 8192 90 | * hard maxlogins 8192 91 | * soft stack -1 92 | * hard stack -1 93 | * soft nproc 2067554 94 | * hard nproc 2067554 95 | * soft nofile 500000 96 | * hard nofile 500000 97 | ' >> /etc/security/limits.conf 98 | 99 | echo ' 100 | ulimit -l unlimited 101 | ulimit -m unlimited 102 | ulimit -c unlimited 103 | ulimit -s unlimited 104 | ulimit -u 2067554 105 | ulimit -n 500000 106 | ' >> ~/.bash_profile 107 | 108 | fi 109 | 110 | cd - 111 | 112 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/install_spectrum_scale.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | cd /tmp/ 4 | curl -O $downloadUrl -s 5 | 6 | while [ $? -ne 0 ]; do 7 | rm -rf /tmp/Spectrum_Scale_Data_Management-* 8 | rm -rf "/tmp/Spectrum Scale*" 9 | curl -O $downloadUrl -s 10 | done 11 | 12 | echo $downloadUrl | grep "Developer" | grep "zip$" 13 | if [ $? -eq 0 ]; then 14 | SS_DE=true 15 | zip_filepath=`ls /tmp/* | grep "Developer" | grep "${version}" | grep "zip$" ` 16 | unzip "$zip_filepath" 17 | install_dir=`ls -d /tmp/*/ | grep "Developer" | grep "Edition" ` 18 | cd """$install_dir""" 19 | cp Spectrum_Scale_Developer-${version}-x86_64-Linux-install /tmp/ 20 | install_filepath="/tmp/Spectrum_Scale_Developer-${version}-x86_64-Linux-install" 21 | else 22 | SS_DE=false 23 | install_filepath="/tmp/Spectrum_Scale_Data_Management-${version}-x86_64-Linux-install" 24 | fi 25 | 26 | while [ ! -f $install_filepath ]; 27 | do 28 | sleep 5s 29 | echo "Waiting for download" 30 | done 31 | 32 | chmod +x $install_filepath 33 | $install_filepath --silent 34 | 35 | 36 | echo "$version" > /etc/yum/vars/spec_scale_ver 37 | 38 | echo '[spectrum_scale-gpfs] 39 | name = Spectrum Scale - GPFS 40 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/gpfs_rpms 41 | gpgcheck=0 42 | enabled=1 43 | [spectrum_scale-ganesha] 44 | name = Spectrum Scale - NFS-Ganesha 45 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/ganesha_rpms/rhel7 46 | gpgcheck=0 47 | enabled=1 48 | [spectrum_scale-smb] 49 | name = Spectrum Scale - SMB 50 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/smb_rpms/rhel7 51 | gpgcheck=0 52 | enabled=1 53 | [spectrum_scale-zimon] 54 | name = Spectrum Scale - Zimon 55 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/zimon_rpms/rhel7 56 | gpgcheck=0 57 | enabled=1 58 | [spectrum_scale-gpfs-optional] 59 | name = Spectrum Scale - GPFS optional 60 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/gpfs_rpms/rhel 61 | gpgcheck=0 62 | enabled=1 63 | ' > /etc/yum.repos.d/spectrum-scale.repo 64 | 65 | echo '[spectrum_scale-object-rhel8] 66 | name = Spectrum Scale - Object 67 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/object_rpms/rhel8 68 | gpgcheck=0 69 | enabled=1 70 | [spectrum_scale-gpfs-rhel] 71 | name = Spectrum Scale - rhel 72 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/gpfs_rpms/rhel 73 | gpgcheck=0 74 | enabled=1 75 | ' >> /etc/yum.repos.d/spectrum-scale.repo 76 | 77 | 78 | yum clean all 79 | yum makecache 80 | rerun=false 81 | yum -y install cpp gcc gcc-c++ binutils 82 | if [ $? -ne 0 ]; then 83 | rerun=true 84 | fi 85 | 86 | function downloadKernelRPMs { 87 | # eg: "kernel-devel" 88 | packagePrefix=$1 89 | kernelVersion=`uname -a | gawk -F" " '{ print $3 }' ` ; echo $kernelVersion 90 | sudo yum install -y -q redhat-lsb-core 91 | lsb_release -a 92 | osVersion=`lsb_release -a | grep "Release:" | gawk -F" " '{ print $2 }' | gawk -F"." '{ print $1"."$2 }' ` ; echo $osVersion 93 | fullOSReleaseVersion=`lsb_release -a | grep "Release:" | gawk -F" " '{ print $2 }'` ; echo $fullOSReleaseVersion 94 | 95 | declare -a rpmServers=("http://linuxsoft.cern.ch/centos-vault/${fullOSReleaseVersion}/updates/x86_64/Packages" 96 | "http://repo1.xorcom.com/repos/centos/7/x86_64/Updates_OS_X86_64/Packages/k" 97 | "http://ftp.scientificlinux.org/linux/scientific/${osVersion}/x86_64/updates/security" 98 | "http://archive.kernel.org/centos-vault/${fullOSReleaseVersion}/updates/x86_64/Packages" 99 | ) 100 | 101 | for rpmDownloadURLPrefix in "${rpmServers[@]}" 102 | do 103 | echo "$rpmDownloadURLPrefix" 104 | curl --head --fail --silent ${rpmDownloadURLPrefix}/${packagePrefix}-${kernelVersion}.rpm 105 | if [ $? -eq 0 ]; then 106 | curl -O ${rpmDownloadURLPrefix}/${packagePrefix}-${kernelVersion}.rpm 107 | if [ $? -eq 0 ]; then 108 | break; 109 | fi 110 | fi 111 | done 112 | } 113 | kernelVersion=`uname -a | gawk -F" " '{ print $3 }' ` ; echo $kernelVersion 114 | downloadKernelRPMs "kernel-devel" 115 | downloadKernelRPMs "kernel-headers" 116 | # --oldpackage 117 | rpm -Uvh kernel-devel-${kernelVersion}.rpm --oldpackage 118 | rpm -Uvh kernel-headers-${kernelVersion}.rpm --oldpackage 119 | 120 | if [ "$rerun" = "true" ]; then 121 | yum -y install cpp gcc gcc-c++ binutils 122 | fi 123 | 124 | yum -y install psmisc numad numactl iperf3 dstat iproute automake autoconf git 125 | 126 | #non-GUI node: 127 | yum -y install gpfs.base gpfs.gpl gpfs.msg.en_US gpfs.gskit gpfs.license* gpfs.ext gpfs.crypto gpfs.compression gpfs.adv gpfs.gss.pmsensors gpfs.docs gpfs.java gpfs.kafka gpfs.librdkafka 128 | 129 | sed -i '/distroverpkg/a exclude=kernel*' /etc/yum.conf 130 | 131 | 132 | echo "cloud-init complete" 133 | touch /tmp/cloud_init.complete 134 | 135 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/nodes-cloud-init-complete-status-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | hostname 4 | while [ ! -f /tmp/cloud_init.complete ] 5 | do 6 | sleep 60s 7 | echo "Waiting for compute node: `hostname --fqdn` cloud-init to complete ..." 8 | done 9 | 10 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/passwordless_ssh.sh: -------------------------------------------------------------------------------- 1 | echo "$sshPrivateKey" > /root/.ssh/id_rsa 2 | echo "$sshPublicKey" > /root/.ssh/id_rsa.pub 3 | chmod 600 ~/.ssh/id_rsa* 4 | chmod 640 ~/.ssh/authorized_keys 5 | 6 | 7 | cp /etc/ssh/sshd_config /etc/ssh/sshd_config.backup 8 | sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config 9 | 10 | mv /root/.ssh/authorized_keys /root/.ssh/authorized_keys.backup 11 | cp /home/opc/.ssh/authorized_keys /root/.ssh/authorized_keys 12 | cd /root/.ssh/; cat id_rsa.pub >> authorized_keys ; cd - 13 | 14 | find_cluster_nodes () { 15 | echo "Doing nslookup for $nodeType nodes" 16 | ct=1 17 | if [ $nodeCount -gt 0 ]; then 18 | while [ $ct -le $nodeCount ]; do 19 | nslk=`nslookup $nodeHostnamePrefix${ct}.$domainName` 20 | ns_ck=`echo -e $?` 21 | if [ $ns_ck = 0 ]; then 22 | hname=`nslookup $nodeHostnamePrefix${ct}.$domainName | grep Name | gawk '{print $2}'` 23 | echo "$hname" >> /tmp/${nodeType}nodehosts; 24 | echo "$hname" >> /tmp/allnodehosts; 25 | ct=$((ct+1)); 26 | else 27 | echo "Sleeping for 10 secs and will check again for nslookup $nodeHostnamePrefix${ct}.$domainName" 28 | sleep 10 29 | fi 30 | done; 31 | echo "Found `cat /tmp/${nodeType}nodehosts | wc -l` $nodeType nodes"; 32 | echo `cat /tmp/${nodeType}nodehosts`; 33 | else 34 | echo "no $nodeType nodes configured" 35 | fi 36 | } 37 | 38 | # Subnet used for GPFS network 39 | domainName=${privateBSubnetsFQDN} 40 | 41 | nodeType="client" 42 | nodeHostnamePrefix=$clientNodeHostnamePrefix 43 | nodeCount=$clientNodeCount 44 | find_cluster_nodes 45 | 46 | 47 | 48 | if [ ! -f ~/.ssh/known_hosts ]; then 49 | touch ~/.ssh/known_hosts 50 | fi 51 | 52 | do_ssh_keyscan () { 53 | if [ -z `ssh-keygen -F $host` ]; then 54 | ssh-keyscan -H $host > /tmp/keyscan 55 | cat /tmp/keyscan | grep "ssh-rsa" 56 | while [ $? -ne 0 ]; do 57 | sleep 10s; 58 | ssh-keyscan -H $host > /tmp/keyscan 59 | cat /tmp/keyscan | grep "ssh-rsa" 60 | done; 61 | ssh-keyscan -H $host >> ~/.ssh/known_hosts 62 | fi 63 | } 64 | 65 | ### passwordless ssh setup 66 | for host_fqdn in `cat /tmp/allnodehosts` ; do 67 | host=$host_fqdn 68 | do_ssh_keyscan 69 | host=${host_fqdn%%.*} 70 | do_ssh_keyscan 71 | host_ip=`nslookup $host_fqdn | grep "Address: " | gawk '{print $2}'` 72 | host=$host_ip 73 | do_ssh_keyscan 74 | # update /etc/hosts file on all nodes with ip, fqdn and hostname of all nodes 75 | echo "$host_ip ${host_fqdn} ${host_fqdn%%.*}" >> /etc/hosts 76 | done ; 77 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/set_env_variables.sh: -------------------------------------------------------------------------------- 1 | 2 | echo " 3 | version=\"$version\" 4 | downloadUrl=\"$downloadUrl\" 5 | sshPrivateKey=\"$sshPrivateKey\" 6 | sshPublicKey=\"$sshPublicKey\" 7 | clientNodeCount=\"$clientNodeCount\" 8 | clientNodeHostnamePrefix=\"$clientNodeHostnamePrefix\" 9 | installerNode=\"$installerNode\" 10 | vcnFQDN=\"$vcnFQDN\" 11 | privateBSubnetsFQDN=\"$privateBSubnetsFQDN\" 12 | " > /tmp/gpfs_env_variables.sh 13 | 14 | # we might need this for BM shapes for core OS services to be ready 15 | sleep 60s 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /clients_only_cluster_scripts/update_resolv_conf.sh: -------------------------------------------------------------------------------- 1 | ## Modify resolv.conf to ensure DNS lookups work from one private subnet to another subnet 2 | mv /etc/resolv.conf /etc/resolv.conf.backup 3 | echo "search ${vcnFQDN} ${privateBSubnetsFQDN} " > /etc/resolv.conf 4 | echo "nameserver 169.254.169.254" >> /etc/resolv.conf 5 | 6 | # The below is to ensure any custom change to hosts and resolv.conf will not be overwritten with data from metaservice, but dhclient will still overwrite resolv.conf. 7 | if [ -z /etc/oci-hostname.conf ]; then 8 | echo "PRESERVE_HOSTINFO=2" > /etc/oci-hostname.conf 9 | else 10 | # https://docs.cloud.oracle.com/iaas/Content/Network/Tasks/managingDHCP.htm#notes 11 | sed -i "s/^PRESERVE_HOSTINFO/#PRESERVE_HOSTINFO/g" /etc/oci-hostname.conf 12 | echo "PRESERVE_HOSTINFO=2" >> /etc/oci-hostname.conf 13 | fi 14 | # The below is to ensure above changes will not be overwritten by dhclient 15 | chattr +i /etc/resolv.conf 16 | -------------------------------------------------------------------------------- /direct_attached/README.md: -------------------------------------------------------------------------------- 1 | # IBM Spectrum Scale on OCI 2 | This Terrrafom template deploys an IBM Spectrum Scale on Oracle Cloud Infrastructure (OCI) using direct attached architecture for data only storage disks. 3 | 4 | 5 | 6 | # High Level Architecture 7 | The template creates all the required infrastucture (virtual network, nat gateway, securitylist, compute, Block volume etc.) as well as installs and configures IBM Spectrum Scale Data Management software. 8 | 9 | The solution can be deployed in a single AD using block volumes. 10 | 11 | 12 | ![](../images/04-direct-attached-architecture.png) 13 | 14 | 15 | ## IBM Spectrum Scale Data Management license 16 | This template assumes you already have purchased a license from IBM and have downloaded the software. The software needs to be stored on a server which is accessible from the servers created by this template in OCI. For example: you can save the software in OCI Object Storage bucket and create pre-authenticated request to use in your template. 17 | 18 | 19 | 20 | ## Prerequisites 21 | First off you'll need to do some pre deploy setup. That's all detailed [here](https://github.com/oracle/oci-quickstart-prerequisites). 22 | 23 | 24 | ## Clone the Terraform template 25 | Now, you'll want a local copy of this repo. You can make that with the commands: 26 | 27 | git clone https://github.com/oci-quickstart/oci-ibm-spectrum-scale.git 28 | cd oci-ibm-spectrum-scale/direct_attached 29 | ls 30 | 31 | 32 | 33 | ## Update variables.tf file (optional) 34 | This is optional, but you can update the variables.tf to change compute shapes to use for servers, # of Meta data NSD disks, # of disks to store data only, Compute nodes and and various other values. 35 | 36 | 37 | ## Deployment & Post Deployment 38 | 39 | Deploy using standard Terraform commands 40 | 41 | terraform init && terraform plan && terraform apply 42 | 43 | 44 | -------------------------------------------------------------------------------- /direct_attached/SharedDataBlockVolume.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | resource "oci_core_volume" "SharedDataBlockVolume" { 4 | count = "${var.SharedData["Count"]}" 5 | 6 | availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[var.AD - 1],"name")}" 7 | compartment_id = "${var.compartment_ocid}" 8 | display_name = "SharedData${count.index+1}" 9 | size_in_gbs = "${var.SharedData["Size"]}" 10 | } 11 | 12 | 13 | -------------------------------------------------------------------------------- /direct_attached/compute.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | resource "oci_core_instance" "ComputeNode" { 4 | count = "${var.ComputeNodeCount}" 5 | availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[var.AD - 1],"name")}" 6 | fault_domain = "FAULT-DOMAIN-${(count.index%2)+1}" 7 | compartment_id = "${var.compartment_ocid}" 8 | display_name = "${var.ComputeNodeHostnamePrefix}${format("%01d", count.index+1)}" 9 | shape = "${var.ComputeNodeShape}" 10 | 11 | create_vnic_details { 12 | subnet_id = local.private_subnet_id 13 | hostname_label = "${var.ComputeNodeHostnamePrefix}${format("%01d", count.index+1)}" 14 | assign_public_ip = "false" 15 | } 16 | 17 | 18 | source_details { 19 | source_type = "image" 20 | source_id = "${var.InstanceImageOCID[var.region]}" 21 | } 22 | 23 | launch_options { 24 | network_type = (length(regexall("VM.Standard.E", var.ComputeNodeShape)) > 0 ? "PARAVIRTUALIZED" : "VFIO") 25 | } 26 | 27 | metadata = { 28 | ssh_authorized_keys = "${var.ssh_public_key}" 29 | #user_data = "${base64encode(data.template_file.boot_script.rendered)}" 30 | user_data = "${base64encode(join("\n", list( 31 | "#!/usr/bin/env bash", 32 | "set -x", 33 | "version=\"${var.ibm_ss_version}\"", 34 | "downloadUrl=\"${var.software_download_url}\"", 35 | "sshPrivateKey=\"${var.ssh_private_key}\"", 36 | "sshPublicKey=\"${var.ssh_public_key}\"", 37 | "clientNodeCount=\"${var.ComputeNodeCount}\"", 38 | "clientNodeHostnamePrefix=\"${var.ComputeNodeHostnamePrefix}\"", 39 | "blockSize=\"${var.BlockSize}\"", 40 | "dataReplica=\"${var.DataReplica}\"", 41 | "metadataReplica=\"${var.metadataReplica}\"", 42 | "gpfsMountPoint=\"${var.GpfsMountPoint}\"", 43 | "fileSystemName=\"${var.fileSystemName}\"", 44 | "sharedDataDiskCount=\"${var.SharedData["Count"]}\"", 45 | "installerNode=\"${var.ComputeNodeHostnamePrefix}${var.installer_node}\"", 46 | "privateSubnetsFQDN=\"${local.private_subnet_domain_name}\"", 47 | "quorumNodeCount=\"${local.derived_quorum_node_count}\"", 48 | "quorumNodeHostnamePrefix=\"${var.QuorumNodeHostnamePrefix}\"", 49 | file("${var.scripts_directory}/boot.sh") 50 | )))}" 51 | } 52 | 53 | 54 | timeouts { 55 | create = "60m" 56 | } 57 | 58 | } 59 | 60 | 61 | 62 | /* bastion instances */ 63 | 64 | resource "oci_core_instance" "bastion" { 65 | count = "${var.BastionNodeCount}" 66 | availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[var.AD - 1],"name")}" 67 | compartment_id = "${var.compartment_ocid}" 68 | display_name = "bastion ${format("%01d", count.index+1)}" 69 | shape = "${var.BastionNodeShape}" 70 | 71 | create_vnic_details { 72 | subnet_id = local.bastion_subnet_id 73 | hostname_label = "bastion-${format("%01d", count.index+1)}" 74 | skip_source_dest_check = true 75 | } 76 | 77 | metadata = { 78 | ssh_authorized_keys = "${var.ssh_public_key}" 79 | } 80 | 81 | 82 | source_details { 83 | source_type = "image" 84 | source_id = "${var.InstanceImageOCID[var.region]}" 85 | } 86 | } 87 | 88 | 89 | 90 | /* Remote exec to deploy gpfs software/rpms on client nodes */ 91 | resource "null_resource" "deploy_gpfs_on_client_nodes" { 92 | depends_on = [ 93 | oci_core_instance.ComputeNode, 94 | oci_core_volume_attachment.ComputeNode_fsd_blockvolume_attach, 95 | oci_core_volume_attachment.ComputeNode_shared_data_blockvolume_attach, 96 | null_resource.notify_compute_nodes_oci_cli_multi_attach_complete, 97 | ] 98 | count = var.ComputeNodeCount 99 | triggers = { 100 | instance_ids = "oci_core_instance.ComputeNode.*.id" 101 | } 102 | 103 | provisioner "file" { 104 | source = "${var.scripts_directory}/nodes-cloud-init-complete-status-check.sh" 105 | destination = "/tmp/nodes-cloud-init-complete-status-check.sh" 106 | connection { 107 | agent = false 108 | timeout = "30m" 109 | host = element(oci_core_instance.ComputeNode.*.private_ip, count.index) 110 | user = var.ssh_user 111 | private_key = var.ssh_private_key 112 | bastion_host = oci_core_instance.bastion[0].public_ip 113 | bastion_port = "22" 114 | bastion_user = var.ssh_user 115 | bastion_private_key = var.ssh_private_key 116 | } 117 | } 118 | 119 | provisioner "file" { 120 | source = "${var.scripts_directory}/deploy_spectrum_scale.sh" 121 | destination = "/tmp/deploy_spectrum_scale.sh" 122 | connection { 123 | agent = false 124 | timeout = "30m" 125 | host = element(oci_core_instance.ComputeNode.*.private_ip, count.index) 126 | user = var.ssh_user 127 | private_key = var.ssh_private_key 128 | bastion_host = oci_core_instance.bastion[0].public_ip 129 | bastion_port = "22" 130 | bastion_user = var.ssh_user 131 | bastion_private_key = var.ssh_private_key 132 | } 133 | } 134 | 135 | provisioner "remote-exec" { 136 | connection { 137 | agent = false 138 | timeout = "30m" 139 | host = element(oci_core_instance.ComputeNode.*.private_ip, count.index) 140 | user = var.ssh_user 141 | private_key = var.ssh_private_key 142 | bastion_host = oci_core_instance.bastion[0].public_ip 143 | bastion_port = "22" 144 | bastion_user = var.ssh_user 145 | bastion_private_key = var.ssh_private_key 146 | } 147 | inline = [ 148 | "set -x", 149 | "echo about to run /tmp/nodes-cloud-init-complete-status-check.sh", 150 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 151 | "sudo -s bash -c 'set -x && /tmp/nodes-cloud-init-complete-status-check.sh'", 152 | "sudo -s bash -c 'set -x && /tmp/deploy_spectrum_scale.sh'", 153 | ] 154 | } 155 | } 156 | 157 | 158 | /* Remote exec to create gpfs cluster on installer node */ 159 | resource "null_resource" "create_gpfs_cluster" { 160 | depends_on = [ 161 | oci_core_instance.ComputeNode, 162 | null_resource.notify_compute_nodes_oci_cli_multi_attach_complete, 163 | null_resource.deploy_gpfs_on_client_nodes, 164 | null_resource.deploy_gpfs_on_quorum_nodes, 165 | ] 166 | count = 1 167 | triggers = { 168 | instance_ids = "oci_core_instance.ComputeNode.*.id" 169 | } 170 | 171 | provisioner "file" { 172 | source = "${var.scripts_directory}/create_spectrum_scale_cluster.sh" 173 | destination = "/tmp/create_spectrum_scale_cluster.sh" 174 | connection { 175 | agent = false 176 | timeout = "30m" 177 | host = element(oci_core_instance.ComputeNode.*.private_ip, count.index) 178 | user = var.ssh_user 179 | private_key = var.ssh_private_key 180 | bastion_host = oci_core_instance.bastion[0].public_ip 181 | bastion_port = "22" 182 | bastion_user = var.ssh_user 183 | bastion_private_key = var.ssh_private_key 184 | } 185 | } 186 | 187 | provisioner "remote-exec" { 188 | connection { 189 | agent = false 190 | timeout = "30m" 191 | host = element(oci_core_instance.ComputeNode.*.private_ip, count.index) 192 | user = var.ssh_user 193 | private_key = var.ssh_private_key 194 | bastion_host = oci_core_instance.bastion[0].public_ip 195 | bastion_port = "22" 196 | bastion_user = var.ssh_user 197 | bastion_private_key = var.ssh_private_key 198 | } 199 | inline = [ 200 | "set -x", 201 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 202 | "sudo su -l -c 'set -x && /tmp/create_spectrum_scale_cluster.sh'", 203 | ] 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /direct_attached/datasources.tf: -------------------------------------------------------------------------------- 1 | # Gets a list of Availability Domains 2 | data "oci_identity_availability_domains" "ADs" { 3 | compartment_id = "${var.tenancy_ocid}" 4 | } 5 | 6 | 7 | data "oci_core_vcn" "vcn" { 8 | vcn_id = var.use_existing_vcn ? var.vcn_id : oci_core_vcn.vcn[0].id 9 | } 10 | 11 | data "oci_core_subnet" "private" { 12 | subnet_id = var.use_existing_vcn ? var.private_subnet_id : local.private_subnet_id 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /direct_attached/multi-attach-storage.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | resource "null_resource" "copy_nsddevices_to_all_compute_nodes" { 5 | depends_on = [oci_core_instance.ComputeNode] 6 | count = var.ComputeNodeCount 7 | provisioner "file" { 8 | source = "../direct_attached_scripts/nsddevices" 9 | destination = "/tmp/nsddevices" 10 | connection { 11 | agent = false 12 | timeout = "30m" 13 | host = "${element(oci_core_instance.ComputeNode.*.private_ip, count.index)}" 14 | user = "${var.ssh_user}" 15 | private_key = "${var.ssh_private_key}" 16 | bastion_host = "${oci_core_instance.bastion.*.public_ip[0]}" 17 | bastion_port = "22" 18 | bastion_user = "${var.ssh_user}" 19 | bastion_private_key = "${var.ssh_private_key}" 20 | } 21 | } 22 | } 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | /* 32 | * 33 | All commented code below - ignore them 34 | * 35 | */ 36 | 37 | 38 | 39 | /* 40 | * 41 | * 42 | resource "null_resource" "install_oci_cli_preview" { 43 | count = "1" 44 | provisioner "local-exec" { 45 | command = "set -x; oci os bucket list --compartment-id ${var.compartment_ocid};" 46 | 47 | 48 | } 49 | } 50 | */ 51 | 52 | 53 | /* 54 | resource "null_resource" "multi_attach_shared_data_bv_to_server_nodes" { 55 | depends_on = ["oci_core_instance.ServerNode" , "oci_core_instance.ComputeNode", "oci_core_volume.SharedDataBlockVolume", "null_resource.install_oci_cli_preview" ] 56 | count = "${var.ServerNodeCount * var.SharedData["Count"]}" 57 | 58 | provisioner "local-exec" { 59 | command = "oci compute volume-attachment attach --type iscsi --is-shareable true --instance-id ${oci_core_instance.ServerNode.*.id[count.index%var.ServerNodeCount]} --volume-id ${oci_core_volume.SharedDataBlockVolume.*.id[floor(count.index/var.ServerNodeCount)]} --device ${var.SharedDataVolumeAttachDeviceMapping[floor(count.index/var.ServerNodeCount)]};sleep 61s; " 60 | } 61 | 62 | } 63 | */ 64 | 65 | /* 66 | * 67 | * 68 | resource "null_resource" "multi_attach_shared_data_bv_to_compute_nodes" { 69 | depends_on = [oci_core_instance.ComputeNode, oci_core_volume.SharedDataBlockVolume, null_resource.install_oci_cli_preview ] 70 | count = "${var.ComputeNodeCount * var.SharedData["Count"]}" 71 | 72 | provisioner "local-exec" { 73 | command = "sleep 45s;oci compute volume-attachment attach --type iscsi --is-shareable true --instance-id ${oci_core_instance.ComputeNode.*.id[count.index%var.ComputeNodeCount]} --volume-id ${oci_core_volume.SharedDataBlockVolume.*.id[floor(count.index/var.ComputeNodeCount)]} --device ${var.SharedDataVolumeAttachDeviceMapping[floor(count.index/var.ComputeNodeCount)]} ; " 74 | } 75 | 76 | } 77 | */ 78 | 79 | /* 80 | resource "null_resource" "multi_attach_shared_metadata_bv_to_server_nodes" { 81 | depends_on = ["oci_core_instance.ServerNode" , "oci_core_instance.ComputeNode", "oci_core_volume.SharedMetaDataBlockVolume", "null_resource.install_oci_cli_preview" ] 82 | count = "${var.ServerNodeCount * var.SharedMetaData["Count"]}" 83 | 84 | provisioner "local-exec" { 85 | command = "sleep 90s; oci compute volume-attachment attach --type iscsi --is-shareable true --instance-id ${oci_core_instance.ServerNode.*.id[count.index%var.ServerNodeCount]} --volume-id ${oci_core_volume.SharedMetaDataBlockVolume.*.id[floor(count.index/var.ServerNodeCount)]} --device ${var.SharedMetaDataVolumeAttachDeviceMapping[floor(count.index/var.ServerNodeCount)]};" 86 | } 87 | 88 | } 89 | 90 | 91 | resource "null_resource" "multi_attach_shared_metadata_bv_to_compute_nodes" { 92 | depends_on = ["oci_core_instance.ServerNode" , "oci_core_instance.ComputeNode", "oci_core_volume.SharedMetaDataBlockVolume", "null_resource.install_oci_cli_preview" ] 93 | count = "${var.ComputeNodeCount * var.SharedMetaData["Count"]}" 94 | } 95 | */ 96 | 97 | /* 98 | * 99 | * 100 | resource "null_resource" "notify_server_nodes_oci_cli_multi_attach_complete" { 101 | depends_on = ["null_resource.multi_attach_shared_metadata_bv_to_compute_nodes" , "null_resource.multi_attach_shared_metadata_bv_to_server_nodes", null_resource.multi_attach_shared_data_bv_to_compute_nodes, "null_resource.multi_attach_shared_data_bv_to_server_nodes" ] 102 | count = "${var.ServerNodeCount}" 103 | provisioner "remote-exec" { 104 | connection { 105 | agent = false 106 | timeout = "30m" 107 | host = "${element(oci_core_instance.ServerNode.*.private_ip, count.index)}" 108 | user = "${var.ssh_user}" 109 | private_key = "${var.ssh_private_key}" 110 | bastion_host = "${oci_core_instance.bastion.*.public_ip[0]}" 111 | bastion_port = "22" 112 | bastion_user = "${var.ssh_user}" 113 | bastion_private_key = "${var.ssh_private_key}" 114 | } 115 | inline = [ 116 | "set -x", 117 | "sudo touch /tmp/multi-attach.complete", 118 | ] 119 | } 120 | } 121 | */ 122 | 123 | 124 | 125 | 126 | -------------------------------------------------------------------------------- /direct_attached/network.tf: -------------------------------------------------------------------------------- 1 | /* 2 | All network resources for this template 3 | */ 4 | /* 5 | resource "oci_core_virtual_network" "ibmss_vcnv3" { 6 | cidr_block = "${var.VPC-CIDR}" 7 | compartment_id = "${var.compartment_ocid}" 8 | display_name = "ibmssvcnv3" 9 | dns_label = "ibmssvcnv3" 10 | } 11 | 12 | resource "oci_core_internet_gateway" "ibmss_internet_gateway" { 13 | compartment_id = "${var.compartment_ocid}" 14 | display_name = "ibmss_internet_gateway" 15 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 16 | } 17 | 18 | resource "oci_core_route_table" "RouteForComplete" { 19 | compartment_id = "${var.compartment_ocid}" 20 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 21 | display_name = "RouteTableForComplete" 22 | route_rules { 23 | cidr_block = "0.0.0.0/0" 24 | network_entity_id = "${oci_core_internet_gateway.ibmss_internet_gateway.id}" 25 | } 26 | } 27 | 28 | 29 | resource "oci_core_nat_gateway" "ibmss_nat_gateway" { 30 | compartment_id = "${var.compartment_ocid}" 31 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 32 | display_name = "ibmss_nat_gateway" 33 | } 34 | 35 | 36 | resource "oci_core_route_table" "PrivateRouteTable" { 37 | compartment_id = "${var.compartment_ocid}" 38 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 39 | display_name = "PrivateRouteTableForComplete" 40 | 41 | route_rules { 42 | destination = "0.0.0.0/0" 43 | network_entity_id = "${oci_core_nat_gateway.ibmss_nat_gateway.id}" 44 | 45 | } 46 | } 47 | 48 | resource "oci_core_security_list" "PublicSubnet" { 49 | compartment_id = "${var.compartment_ocid}" 50 | display_name = "Public Subnet" 51 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 52 | egress_security_rules { 53 | destination = "0.0.0.0/0" 54 | protocol = "6" 55 | } 56 | 57 | ingress_security_rules { 58 | tcp_options { 59 | max = 22 60 | min = 22 61 | } 62 | protocol = "6" 63 | source = "0.0.0.0/0" 64 | } 65 | 66 | 67 | } 68 | 69 | 70 | 71 | resource "oci_core_security_list" "PrivateSubnet" { 72 | compartment_id = "${var.compartment_ocid}" 73 | display_name = "Private" 74 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 75 | 76 | egress_security_rules { 77 | destination = "0.0.0.0/0" 78 | protocol = "all" 79 | } 80 | 81 | egress_security_rules { 82 | protocol = "all" 83 | destination = "${var.VPC-CIDR}" 84 | } 85 | ingress_security_rules { 86 | tcp_options { 87 | max = 22 88 | min = 22 89 | } 90 | protocol = "6" 91 | source = "${var.VPC-CIDR}" 92 | } 93 | ingress_security_rules { 94 | protocol = "All" 95 | source = "${var.VPC-CIDR}" 96 | } 97 | 98 | } 99 | 100 | 101 | 102 | 103 | ## Publicly Accessable Subnet Setup 104 | 105 | resource "oci_core_subnet" "public" { 106 | count = "3" 107 | availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[count.index%3],"name")}" 108 | cidr_block = "${cidrsubnet(var.VPC-CIDR, 8, count.index)}" 109 | display_name = "public_${count.index}" 110 | compartment_id = "${var.compartment_ocid}" 111 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 112 | route_table_id = "${oci_core_route_table.RouteForComplete.id}" 113 | security_list_ids = ["${oci_core_security_list.PublicSubnet.id}"] 114 | dhcp_options_id = "${oci_core_virtual_network.ibmss_vcnv3.default_dhcp_options_id}" 115 | dns_label = "public${count.index}" 116 | } 117 | 118 | ## Private Subnet Setup 119 | 120 | resource "oci_core_subnet" "private" { 121 | count = "3" 122 | availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[count.index%3],"name")}" 123 | cidr_block = "${cidrsubnet(var.VPC-CIDR, 8, count.index+3)}" 124 | display_name = "private_${count.index}" 125 | compartment_id = "${var.compartment_ocid}" 126 | vcn_id = "${oci_core_virtual_network.ibmss_vcnv3.id}" 127 | route_table_id = "${oci_core_route_table.PrivateRouteTable.id}" 128 | security_list_ids = ["${oci_core_security_list.PrivateSubnet.id}"] 129 | dhcp_options_id = "${oci_core_virtual_network.ibmss_vcnv3.default_dhcp_options_id}" 130 | prohibit_public_ip_on_vnic = "true" 131 | dns_label = "private${count.index}" 132 | } 133 | */ 134 | 135 | /* 136 | All network resources for this template 137 | */ 138 | 139 | resource "oci_core_vcn" "vcn" { 140 | count = var.use_existing_vcn ? 0 : 1 141 | cidr_block = var.vpc_cidr 142 | compartment_id = var.compartment_ocid 143 | display_name = "gpfs" 144 | dns_label = "gpfs" 145 | } 146 | 147 | resource "oci_core_internet_gateway" "internet_gateway" { 148 | count = var.use_existing_vcn ? 0 : 1 149 | compartment_id = var.compartment_ocid 150 | display_name = "internet_gateway" 151 | vcn_id = oci_core_vcn.vcn[0].id 152 | } 153 | 154 | resource "oci_core_route_table" "pubic_route_table" { 155 | count = var.use_existing_vcn ? 0 : 1 156 | compartment_id = var.compartment_ocid 157 | vcn_id = oci_core_vcn.vcn[0].id 158 | display_name = "RouteTableForComplete" 159 | route_rules { 160 | cidr_block = "0.0.0.0/0" 161 | network_entity_id = oci_core_internet_gateway.internet_gateway[0].id 162 | } 163 | } 164 | 165 | 166 | resource "oci_core_nat_gateway" "nat_gateway" { 167 | count = var.use_existing_vcn ? 0 : 1 168 | compartment_id = var.compartment_ocid 169 | vcn_id = oci_core_vcn.vcn[0].id 170 | display_name = "nat_gateway" 171 | } 172 | 173 | 174 | resource "oci_core_route_table" "private_route_table" { 175 | count = var.use_existing_vcn ? 0 : 1 176 | compartment_id = var.compartment_ocid 177 | vcn_id = oci_core_vcn.vcn[0].id 178 | display_name = "private_route_tableForComplete" 179 | route_rules { 180 | destination = "0.0.0.0/0" 181 | network_entity_id = oci_core_nat_gateway.nat_gateway[0].id 182 | } 183 | } 184 | 185 | resource "oci_core_security_list" "public_security_list" { 186 | count = var.use_existing_vcn ? 0 : 1 187 | compartment_id = var.compartment_ocid 188 | display_name = "Public Subnet" 189 | vcn_id = oci_core_vcn.vcn[0].id 190 | egress_security_rules { 191 | destination = "0.0.0.0/0" 192 | protocol = "6" 193 | } 194 | ingress_security_rules { 195 | tcp_options { 196 | max = 22 197 | min = 22 198 | } 199 | protocol = "6" 200 | source = "0.0.0.0/0" 201 | } 202 | ingress_security_rules { 203 | tcp_options { 204 | max = 3389 205 | min = 3389 206 | } 207 | protocol = "6" 208 | source = "0.0.0.0/0" 209 | } 210 | } 211 | 212 | # https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewall.htm 213 | resource "oci_core_security_list" "private_security_list" { 214 | count = var.use_existing_vcn ? 0 : 1 215 | compartment_id = var.compartment_ocid 216 | display_name = "Private" 217 | vcn_id = oci_core_vcn.vcn[0].id 218 | 219 | egress_security_rules { 220 | destination = "0.0.0.0/0" 221 | protocol = "all" 222 | } 223 | egress_security_rules { 224 | protocol = "all" 225 | destination = var.vpc_cidr 226 | } 227 | ingress_security_rules { 228 | tcp_options { 229 | max = 22 230 | min = 22 231 | } 232 | protocol = "6" 233 | source = var.vpc_cidr 234 | } 235 | 236 | ingress_security_rules { 237 | protocol = "All" 238 | source = var.vpc_cidr 239 | } 240 | } 241 | 242 | 243 | # Regional subnet - public 244 | resource "oci_core_subnet" "public" { 245 | count = var.use_existing_vcn ? 0 : 1 246 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index) 247 | display_name = "Public-Subnet" 248 | compartment_id = var.compartment_ocid 249 | vcn_id = oci_core_vcn.vcn[0].id 250 | route_table_id = oci_core_route_table.pubic_route_table[0].id 251 | security_list_ids = [oci_core_security_list.public_security_list[0].id] 252 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 253 | dns_label = "public" 254 | } 255 | 256 | # Regional subnet - private 257 | resource "oci_core_subnet" "private" { 258 | count = var.use_existing_vcn ? 0 : 1 259 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index+3) 260 | display_name = "Private-Subnet" 261 | compartment_id = var.compartment_ocid 262 | vcn_id = oci_core_vcn.vcn[0].id 263 | route_table_id = oci_core_route_table.private_route_table[0].id 264 | security_list_ids = [oci_core_security_list.private_security_list[0].id] 265 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 266 | prohibit_public_ip_on_vnic = "true" 267 | dns_label = "private" 268 | } 269 | 270 | 271 | 272 | 273 | 274 | 275 | -------------------------------------------------------------------------------- /direct_attached/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | output "SSH_login" { 4 | value = < 2 ? 0 : 1) 186 | } 187 | 188 | -------------------------------------------------------------------------------- /direct_attached_scripts/create_spectrum_scale_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | source /tmp/gpfs_env_variables.sh 6 | 7 | # Only on installerNode 8 | echo "$thisHost" | grep -q -w $installerNode 9 | if [ $? -eq 0 ] ; then 10 | rm /root/node.stanza 11 | count=0 12 | for hname in `cat /tmp/allnodehosts` ; do 13 | if [ $count -lt 3 ] ; then 14 | if [ $count -eq 2 ] ; then 15 | echo "${hname}" | grep -q $quorumNodeHostnamePrefix 16 | if [ $? -eq 0 ] ; then 17 | echo "${hname}:quorum" >> /root/node.stanza 18 | else 19 | echo "${hname}:quorum-manager" >> /root/node.stanza 20 | fi 21 | else 22 | echo "${hname}:quorum-manager" >> /root/node.stanza 23 | fi 24 | else 25 | echo "${hname}" >> /root/node.stanza 26 | fi 27 | count=$((count+1)) 28 | done 29 | cat /root/node.stanza 30 | 31 | mmcrcluster -N node.stanza -r /usr/bin/ssh -R /usr/bin/scp -C ss-direct-attached -A 32 | sleep 30s 33 | 34 | count=0 35 | for hname in `cat /tmp/allnodehosts` ; do 36 | if [ $count -lt 3 ] ; then 37 | mmchlicense server --accept -N ${hname} 38 | else 39 | mmchlicense client --accept -N ${hname} 40 | fi 41 | count=$((count+1)) 42 | done 43 | 44 | mmlscluster 45 | sleep 30s 46 | 47 | /usr/lpp/mmfs/bin/mmstartup -a 48 | while [ `mmgetstate -a | grep "active" | wc -l` -lt $((clientNodeCount + quorumNodeCount)) ] ; do echo "waiting for client nodes of cluster to start ..." ; sleep 10s; done; 49 | 50 | 51 | 52 | diskArray=(b c d e f g h i j k l m n o p q r s t u v w x y z aa ab ac ad ae af ag) 53 | 54 | stanzaForNSDFileName="/tmp/nsd.stanza" 55 | rm -rf $stanzaForNSDFileName 56 | 57 | for i in `seq 1 $sharedDataDiskCount`; 58 | do 59 | 60 | if ([ $metadataReplica -gt 1 ] || [ $dataReplica -gt 1 ]); then 61 | echo $i 62 | if [ $((i % 2)) -eq 0 ]; then 63 | failureGroup=102 64 | else 65 | failureGroup=101 66 | fi 67 | else 68 | failureGroup=100 69 | fi 70 | echo $failureGroup 71 | 72 | echo " " >> $stanzaForNSDFileName 73 | echo "%nsd: nsd=nsd${i}" >> $stanzaForNSDFileName 74 | echo "device=/dev/oracleoci/oraclevd${diskArray[(($i-1))]}" >> $stanzaForNSDFileName 75 | echo "usage=dataAndMetadata" >> $stanzaForNSDFileName 76 | echo "failureGroup=$failureGroup" >> $stanzaForNSDFileName 77 | echo "pool=system" >> $stanzaForNSDFileName 78 | 79 | done 80 | 81 | if [ $quorumNodeCount -eq 1 ]; then 82 | echo " " >> $stanzaForNSDFileName 83 | echo "%nsd: nsd=nsdquorum" >> $stanzaForNSDFileName 84 | echo "device=/dev/oracleoci/oraclevd${diskArray[(($sharedDataDiskCount))]}" >> $stanzaForNSDFileName 85 | echo "servers=${quorumNodeHostnamePrefix}1" >> $stanzaForNSDFileName 86 | echo "usage=descOnly" >> $stanzaForNSDFileName 87 | echo "failureGroup=500" >> $stanzaForNSDFileName 88 | echo "pool=system" >> $stanzaForNSDFileName 89 | fi 90 | 91 | 92 | mmlsnodeclass --all 93 | 94 | mmcrnsd -F $stanzaForNSDFileName 95 | sleep 15s 96 | /usr/lpp/mmfs/bin/mmlsnsd -X 97 | /usr/lpp/mmfs/bin/mmlsnsd 98 | 99 | mmcrfs fs1 -F $stanzaForNSDFileName -B $blockSize -m $metadataReplica -M 2 -r $dataReplica -R 2 100 | sleep 60s 101 | 102 | mmlsfs fs1 103 | mmlsnsd 104 | mmlsdisk fs1 -L 105 | 106 | mmmount fs1 -a 107 | sleep 15s 108 | df -h 109 | 110 | # Try - 2 FG and roundrobin the NSD in 2 FG , then try metadataReplica=2 111 | 112 | fi 113 | 114 | exit 0; 115 | 116 | 117 | -------------------------------------------------------------------------------- /direct_attached_scripts/deploy_spectrum_scale.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | source /tmp/gpfs_env_variables.sh 6 | 7 | 8 | # Build the GPFS potability layer. 9 | /usr/lpp/mmfs/bin/mmbuildgpl 10 | # have seen the above fail sometimes, hence the below loop 11 | while [ $? -ne 0 ]; do 12 | sleep 10s; 13 | /usr/lpp/mmfs/bin/mmbuildgpl 14 | done; 15 | 16 | 17 | # Up date the PATH environmental variable. 18 | echo -e '\nexport PATH=/usr/lpp/mmfs/bin:$PATH' >> ~/.bash_profile 19 | source ~/.bash_profile 20 | 21 | exit 0; 22 | 23 | 24 | -------------------------------------------------------------------------------- /direct_attached_scripts/nodes-cloud-init-complete-status-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | hostname 4 | while [ ! -f /tmp/cloud_init.complete ] 5 | do 6 | sleep 60s 7 | echo "Waiting for compute node: `hostname --fqdn` cloud-init to complete ..." 8 | done 9 | 10 | 11 | -------------------------------------------------------------------------------- /direct_attached_scripts/nsddevices: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | # @(#)53 1.5 src/avs/fs/mmfs/ts/config/nsddevices.sample, mmfs, avs_rfks2, rfks21523a 8/9/04 16:51:23 3 | ############################################################################## 4 | # 5 | # When properly installed, this script is invoked by the 6 | # /usr/lpp/mmfs/bin/mmdevdiscover script. 7 | # 8 | # INSTALLATION GUIDELINES FOR THIS SCRIPT: 9 | # 10 | # a) edit this script using the configuration guidelines below 11 | # b) copy this script to /var/mmfs/etc/nsddevices 12 | # c) ensure this script is executable (chmod +x /var/mmfs/etc/nsddevices) 13 | # 14 | # DESCRIPTION OF NSD DEVICE DISCOVERY: 15 | # 16 | # The mmdevdiscover script and conversely this script are invoked by 17 | # the mmfsd daemon when it tries to discover or verify physical 18 | # devices previously defined to GPFS with the mmcrnsd command. These 19 | # scripts identify devices found in the /dev file system on the local 20 | # machine that may correlate to NSDs defined to GPFS. 21 | # 22 | # GPFS uses the list of devices output by these scripts in mapping 23 | # the NSD name listed in the configuration database to a local device 24 | # in the /dev file system. When an NSD is created via the mmcrnsd 25 | # command it is marked with a unique identifier written to sector 26 | # two of the device. This unique identifier is recorded in the 27 | # configuration database along with the user recognizable NSD name. 28 | # 29 | # During GPFS disk discovery each device name output by mmdevdiscover 30 | # and nsddevices is opened in turn and sector two of each device is 31 | # read. If a match between an NSD identifier on the device and an 32 | # identifier recorded in the configuration database is found, then 33 | # this machine has local access to the NSD device. I/O is thus 34 | # subsequently performed via this local /dev interface. 35 | # 36 | # CONFIGURATION AND EDITING GUIDELINES: 37 | # 38 | # If this script is not installed then disk discovery is done 39 | # only via the commands listed in mmdevdiscover. 40 | # 41 | # If this script is installed and returns a NON ZERO return code 42 | # then the disk discovery commands listed in mmdevdiscover will ALSO 43 | # be run. 44 | # 45 | # If this script is installed and returns a ZERO return code 46 | # then the disk discovery commands listed in mmdevdiscover will NOT 47 | # be run. 48 | # 49 | # The output from both this script and nsddevices is a number 50 | # of lines in the following format: 51 | # 52 | # deviceName deviceType 53 | # 54 | # where (deviceName) is a device name such as (hdisk1) 55 | # and (deviceType) is a set of known disk types. Consult 56 | # 57 | # /usr/lpp/mmfs/bin/mmdevdiscover 58 | # 59 | # for a list of currently known deviceTypes 60 | # 61 | # Example output: 62 | # 63 | # hdisk1 hdisk 64 | # hdisk2 hdisk 65 | # 66 | ############################################################################## 67 | 68 | osName=$(/bin/uname -s) 69 | 70 | if [[ $osName = Linux ]] 71 | then 72 | #ls /dev/mapper|grep map|awk '{print "mapper/"$1,"dmm"}' 73 | ls /dev/oracleoci | grep oraclevd| awk '{print "oracleoci/"$1,"dmm"}' 74 | fi 75 | 76 | if [[ $osName = AIX ]] 77 | then 78 | : # Add function to discover disks in the AIX environment. 79 | fi 80 | 81 | # To bypass the GPFS disk discovery (/usr/lpp/mmfs/bin/mmdevdiscover), 82 | return 0 83 | 84 | # To continue with the GPFS disk discovery steps, 85 | # return 1 86 | -------------------------------------------------------------------------------- /images/01-shared-nothing-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/01-shared-nothing-architecture.png -------------------------------------------------------------------------------- /images/02-tf-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/02-tf-apply.png -------------------------------------------------------------------------------- /images/03-mm-commands.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/03-mm-commands.png -------------------------------------------------------------------------------- /images/04-direct-attached-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/04-direct-attached-architecture.png -------------------------------------------------------------------------------- /images/multi_cluster_spectrum_scale/01_multi_cluster_spectrum_scale_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/multi_cluster_spectrum_scale/01_multi_cluster_spectrum_scale_architecture.png -------------------------------------------------------------------------------- /images/multi_cluster_spectrum_scale/02_multi_cluster_spectrum_scale_vcn_peering_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/multi_cluster_spectrum_scale/02_multi_cluster_spectrum_scale_vcn_peering_architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/01-high-level-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/01-high-level-architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/01-single-AD-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/01-single-AD-architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/01-two-AD-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/01-two-AD-architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/01a-single-AD-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/01a-single-AD-architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/01a-two-AD-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/01a-two-AD-architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/01b-single-AD-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/01b-single-AD-architecture.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/02-tf-apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/02-tf-apply.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/03-mm-commands.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/03-mm-commands.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/04-gui-charts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/04-gui-charts.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/05-gui-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/05-gui-dashboard.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/06-gui-throughput.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/06-gui-throughput.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss1.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss10.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss11_apply_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss11_apply_output.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss2.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss3.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss4.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss5.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss6.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss7.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss8.png -------------------------------------------------------------------------------- /images/network_shared_disk_server_model/ss9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/images/network_shared_disk_server_model/ss9.png -------------------------------------------------------------------------------- /network_shared_disk_server_model.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/network_shared_disk_server_model.zip -------------------------------------------------------------------------------- /network_shared_disk_server_model/README.md: -------------------------------------------------------------------------------- 1 | # IBM Spectrum Scale on OCI 2 | This Terrrafom template deploys an IBM Spectrum Scale distributed parallel file system on Oracle Cloud Infrastructure (OCI) using Network Shared Disk (NSD) Server model architecture. 3 | 4 | 5 | ## Network Shared Disk (NSD) Server model architecture 6 | The template creates all the required infrastucture (virtual network, nat gateway, securitylist, compute, Block volume etc.) as well as installs and configures IBM Spectrum Scale Data Management software. The solution can be deployed across 2 Availability domains (AD) (set DataReplica parameter to 2) or in a single AD. 7 | 8 | ### Single AD 9 | ![](../images/network_shared_disk_server_model/01b-single-AD-architecture.png) 10 | 11 | ## Multiple AD 12 | We can deploy a stretched cluster across multiple ADs or seperate cluster in each AD and use AFM or other rsync tools to move data. 13 | 14 | 15 | 16 | 17 | ## Prerequisites 18 | 19 | ### 1. IBM Spectrum Scale Software Download 20 | This template is designed to work with the following editions: 21 | 22 | Spectrum Scale Data Management Edition 23 | Spectrum Scale Developer Edition - Free for upto 12TB Storage 24 | Spectrum Scale Data Access Edition 25 | 26 | 27 | Please download the Free developer edition of Spectrum Scale software binary from [IBM website](https://www.ibm.com/sg-en/marketplace/scale-out-file-and-object-storage/purchase). The software needs to be stored on a server which is accessible from the file servers created by this template in OCI. For example: you can save the software in private secure OCI Object Storage bucket and create pre-authenticated request to use in your template. 28 | 29 | If you already have license for Spectrum Scale, then you can download it from [here](https://www.ibm.com/support/fixcentral/swg/selectFixes?parent=Software%20defined%20storage&product=ibm/StorageSoftware/IBM+Spectrum+Scale&release=All&platform=Linux+64-bit,x86_64&function=all) 30 | 31 | 32 | ## Resource Manager Deployment 33 | This Quick Start uses [OCI Resource Manager](https://docs.cloud.oracle.com/iaas/Content/ResourceManager/Concepts/resourcemanager.htm) to make deployment easy, sign up for an [OCI account](https://cloud.oracle.com/en_US/tryit) if you don't have one, and just click the button below: 34 | 35 | [![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://console.us-ashburn-1.oraclecloud.com/resourcemanager/stacks/create?region=home&zipUrl=https://github.com/oracle-quickstart/oci-ibm-spectrum-scale/raw/master/network_shared_disk_server_model_latest.zip) 36 | 37 | Note, if you use this template to create another repo you'll need to change the link for the button to point at your repo. 38 | 39 | 40 | ## Local Terraform Deployment 41 | First off you'll need to do some pre deploy setup on your local machine to use Terraform with Oracle Cloud Infrastructure. That's all detailed [here](https://github.com/oracle/oci-quickstart-prerequisites). 42 | 43 | 44 | ## Clone the Terraform template 45 | Now, you'll want a local copy of this repo. You can make that with the commands: 46 | 47 | git clone https://github.com/oracle/oci-quickstart-ibm-spectrum-scale.git 48 | cd oci-quickstart-ibm-spectrum-scale/network_shared_disk_server_model 49 | ls 50 | 51 | 52 | 53 | ## Update variables.tf file (optional) 54 | Update the variables.tf to provide the link to Spectrum Scale binary software & its version, change compute shapes to use for NSD servers, dataReplica, # of NSD disks, # of NSD and client nodes and and various other values. See [OCI Compute shapes](https://docs.cloud.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm) for more details. 55 | 56 | | Node Type | Mandatory | Node Shape (Recommended) | Node Count (Production minimum) | Node Shape (Minimum) | Node Count (Minimum) | Comments | 57 | | :---: | :---: | :---: | :---: | :---: | :---: | :---: | 58 | | `NSD server` | Yes | BM.Standard2.52 (Recommended) | 2 | Any Intel OCI Compute shape | 2 | Intel only. Bare metal nodes with 2 physical NIC's are recommended for Production. | 59 | | `CES server` | No | BM.Standard2.52 | 2 | Any Intel OCI Compute shape | 0 | Intel Only. Use 1 for testing, 2 for prod, but this node is optional. Bare metal nodes with 2 physical NIC's recommended for Production. Use only if access via NFS, SMB, Object access and Transparent Cloud Tiering is required | 60 | | `MGMT GUI server` | No | VM.Standard2.16 or higher | 1 | Any Intel OCI Compute shape | 0 | Intel Only. Add 2, if you want HA for mgmt GUI node | 61 | | `Client server` | Mandatory | Baremetal or VM - Standard or DenseIO. See [OCI Compute shapes](https://docs.cloud.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm) | 1 | Any OCI Compute shape | 1 | Throughput received will depend on shape selected. You can have many clients | 62 | | `Bastion server` | Mandatory | VM.Standard2.1 / VM.Standard.E2.1 or higher | 1 | - | 1 | Required | 63 | | `Windows SMB client` | No | VM.Standard2.4 | 1 | VM.Standard2.4 | 0 | You can create a node using the Template, but by default, its not created. (Optional) | 64 | | `NFS client` | No | Any Compute shape | 1 | - | 0 | For testing, you can use Bastion node as your NFS client | 65 | | `TCT server` | No | Any Compute shape | 1 | - | 0 | Reach out to OCI HPC team or your account representative. | 66 | 67 | 68 | 69 | 70 | ## Deployment & Post Deployment 71 | 72 | Deploy using standard Terraform commands 73 | 74 | terraform init && terraform plan 75 | terraform apply 76 | 77 | For large clusters, use the below to ensure deployment does not fail due to API throttle limits 78 | 79 | terraform apply -parallelism=3 80 | 81 | 82 | 83 | ## Terraform apply - output 84 | 85 | ![](../images/network_shared_disk_server_model/02-tf-apply.png) 86 | 87 | ## Output for various GPFS commands 88 | 89 | ![](../images/network_shared_disk_server_model/03-mm-commands.png) 90 | 91 | ## Spectrum Scale Management GUI Interface 92 | 93 | ### Metrics 94 | ![](../images/network_shared_disk_server_model/04-gui-charts.png) 95 | 96 | ### Dashboard 97 | ![](../images/network_shared_disk_server_model/05-gui-dashboard.png) 98 | 99 | ### Throughput 100 | ![](../images/network_shared_disk_server_model/06-gui-throughput.png) 101 | 102 | 103 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/ces.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_instance" "ces_node" { 2 | count = var.ces_node_count 3 | availability_domain = local.ad 4 | 5 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 6 | compartment_id = var.compartment_ocid 7 | display_name = "${var.ces_node_hostname_prefix}ptcl-${format("%01d", count.index+1)}" 8 | shape = (local.dual_nics_ces_node ? var.ces_node_shape : var.ces_node_shape) 9 | 10 | create_vnic_details { 11 | subnet_id = oci_core_subnet.protocol_subnet.*.id[0] 12 | hostname_label = "${var.ces_node_hostname_prefix}ptcl-${format("%01d", count.index+1)}" 13 | assign_public_ip = "false" 14 | } 15 | 16 | source_details { 17 | source_type = "image" 18 | source_id = var.images[var.region] 19 | } 20 | 21 | metadata = { 22 | ssh_authorized_keys = tls_private_key.ssh.public_key_openssh 23 | # ssh_authorized_keys = "${var.ssh_public_key}\n${tls_private_key.ssh.public_key_openssh}" 24 | user_data = base64encode(join("\n", list( 25 | "#!/usr/bin/env bash", 26 | "set -x", 27 | "version=\"${var.spectrum_scale_version}\"", 28 | "downloadUrl=\"${var.spectrum_scale_download_url}\"", 29 | "totalNsdNodePools=\"${var.total_nsd_node_pools}\"", 30 | "nsdNodesPerPool=\"${var.nsd_nodes_per_pool}\"", 31 | "nsdNodeCount=\"${(var.total_nsd_node_pools * var.nsd_nodes_per_pool)}\"", 32 | "nsdNodeHostnamePrefix=\"${var.nsd_node_hostname_prefix}\"", 33 | "clientNodeCount=\"${var.client_node_count}\"", 34 | "clientNodeHostnamePrefix=\"${var.client_node_hostname_prefix}\"", 35 | "blockSize=\"${var.spectrum_scale_block_size}\"", 36 | "dataReplica=\"${var.spectrum_scale_data_replica}\"", 37 | "metadataReplica=\"${var.spectrum_scale_metadata_replica}\"", 38 | "gpfsMountPoint=\"${var.spectrum_scale_gpfs_mount_point}\"", 39 | "sharedDataDiskCount=\"${(var.total_nsd_node_pools * var.block_volumes_per_pool)}\"", 40 | "blockVolumesPerPool=\"${var.block_volumes_per_pool}\"", 41 | "installerNode=\"${var.nsd_node_hostname_prefix}${var.installer_node}\"", 42 | "vcnFQDN=\"${local.vcn_domain_name}\"", 43 | "privateSubnetsFQDN=\"${local.storage_subnet_domain_name}\"", 44 | "privateBSubnetsFQDN=\"${local.filesystem_subnet_domain_name}\"", 45 | "cesNodeCount=\"${var.ces_node_count}\"", 46 | "cesNodeHostnamePrefix=\"${var.ces_node_hostname_prefix}\"", 47 | "mgmtGuiNodeCount=\"${var.mgmt_gui_node_count}\"", 48 | "mgmtGuiNodeHostnamePrefix=\"${var.mgmt_gui_node_hostname_prefix}\"", 49 | "privateProtocolSubnetFQDN=\"${local.protocol_subnet_domain_name}\"", 50 | file("${var.scripts_directory}/firewall.sh"), 51 | file("${var.scripts_directory}/set_env_variables.sh"), 52 | file("${var.scripts_directory}/update_resolv_conf.sh"), 53 | file("${var.scripts_directory}/configure_nic.sh"), 54 | file("${var.scripts_directory}/block_volume_discovery.sh"), 55 | file("${var.scripts_directory}/infra_tuning.sh"), 56 | file("${var.scripts_directory}/passwordless_ssh.sh"), 57 | file("${var.scripts_directory}/install_spectrum_scale.sh") 58 | ))) 59 | } 60 | 61 | timeouts { 62 | create = "120m" 63 | } 64 | 65 | } 66 | 67 | 68 | resource "null_resource" "deploy_ssh_keys_on_ces_nodes" { 69 | depends_on = [ 70 | oci_core_instance.ces_node, 71 | ] 72 | count = var.ces_node_count 73 | triggers = { 74 | instance_ids = "oci_core_instance.ces_node.*.id" 75 | } 76 | 77 | provisioner "file" { 78 | content = tls_private_key.ssh.private_key_pem 79 | destination = "/home/opc/.ssh/id_rsa" 80 | connection { 81 | agent = false 82 | timeout = "30m" 83 | host = element(oci_core_instance.ces_node.*.private_ip, count.index) 84 | user = var.ssh_user 85 | private_key = tls_private_key.ssh.private_key_pem 86 | bastion_host = oci_core_instance.bastion[0].public_ip 87 | bastion_port = "22" 88 | bastion_user = var.ssh_user 89 | bastion_private_key = tls_private_key.ssh.private_key_pem 90 | } 91 | } 92 | 93 | provisioner "file" { 94 | content = tls_private_key.ssh.public_key_openssh 95 | destination = "/home/opc/.ssh/id_rsa.pub" 96 | connection { 97 | agent = false 98 | timeout = "30m" 99 | host = element(oci_core_instance.ces_node.*.private_ip, count.index) 100 | user = var.ssh_user 101 | private_key = tls_private_key.ssh.private_key_pem 102 | bastion_host = oci_core_instance.bastion[0].public_ip 103 | bastion_port = "22" 104 | bastion_user = var.ssh_user 105 | bastion_private_key = tls_private_key.ssh.private_key_pem 106 | } 107 | } 108 | 109 | } 110 | 111 | 112 | /* Remote exec to deploy gpfs software/rpms on ces nodes */ 113 | resource "null_resource" "deploy_gpfs_on_ces_nodes" { 114 | depends_on = [ 115 | oci_core_instance.ces_node ] 116 | count = var.ces_node_count 117 | triggers = { 118 | instance_ids = "oci_core_instance.ces_node.*.id" 119 | } 120 | 121 | provisioner "file" { 122 | source = "${var.scripts_directory}/" 123 | destination = "/tmp/" 124 | connection { 125 | agent = false 126 | timeout = "30m" 127 | host = element(oci_core_instance.ces_node.*.private_ip, count.index) 128 | user = var.ssh_user 129 | private_key = tls_private_key.ssh.private_key_pem 130 | bastion_host = oci_core_instance.bastion[0].public_ip 131 | bastion_port = "22" 132 | bastion_user = var.ssh_user 133 | bastion_private_key = tls_private_key.ssh.private_key_pem 134 | } 135 | } 136 | 137 | provisioner "remote-exec" { 138 | connection { 139 | agent = false 140 | timeout = "30m" 141 | host = element(oci_core_instance.ces_node.*.private_ip, count.index) 142 | user = var.ssh_user 143 | private_key = tls_private_key.ssh.private_key_pem 144 | bastion_host = oci_core_instance.bastion[0].public_ip 145 | bastion_port = "22" 146 | bastion_user = var.ssh_user 147 | bastion_private_key = tls_private_key.ssh.private_key_pem 148 | } 149 | inline = [ 150 | "set -x", 151 | "echo about to run /tmp/nodes-cloud-init-complete-status-check.sh", 152 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 153 | "sudo -s bash -c 'set -x && /tmp/nodes-cloud-init-complete-status-check.sh'", 154 | "sudo -s bash -c 'set -x && /tmp/deploy_spectrum_scale.sh'", 155 | ] 156 | } 157 | } 158 | 159 | 160 | 161 | /* Remote exec to configure ces service on ss-ces-1 node */ 162 | resource "null_resource" "configure_ces_service" { 163 | depends_on = [ 164 | oci_core_instance.ces_node, 165 | null_resource.create_gpfs_cluster 166 | ] 167 | count = var.ces_node_count > 0 ? 1 : 0 168 | # 1 169 | triggers = { 170 | instance_ids = element(concat(oci_core_instance.ces_node.*.id, [""]), 0) 171 | } 172 | 173 | provisioner "remote-exec" { 174 | connection { 175 | agent = false 176 | timeout = "30m" 177 | host = element(oci_core_instance.ces_node.*.private_ip, count.index) 178 | user = var.ssh_user 179 | private_key = tls_private_key.ssh.private_key_pem 180 | bastion_host = oci_core_instance.bastion[0].public_ip 181 | bastion_port = "22" 182 | bastion_user = var.ssh_user 183 | bastion_private_key = tls_private_key.ssh.private_key_pem 184 | } 185 | inline = [ 186 | "set -x", 187 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 188 | "sudo su -l -c 'set -x && /tmp/configure_ces.sh'", 189 | ] 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/compute_gpfs_management_gui.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "oci_core_instance" "mgmt_gui_node" { 3 | count = var.mgmt_gui_node_count 4 | availability_domain = local.ad 5 | 6 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 7 | compartment_id = var.compartment_ocid 8 | display_name = "${var.mgmt_gui_node_hostname_prefix}${format("%01d", count.index+1)}" 9 | shape = var.mgmt_gui_node_shape 10 | 11 | create_vnic_details { 12 | subnet_id = local.client_subnet_id 13 | hostname_label = "${var.mgmt_gui_node_hostname_prefix}${format("%01d", count.index+1)}" 14 | assign_public_ip = "false" 15 | } 16 | 17 | source_details { 18 | source_type = "image" 19 | source_id = var.images[var.region] 20 | } 21 | 22 | metadata = { 23 | ssh_authorized_keys = tls_private_key.ssh.public_key_openssh 24 | # ssh_authorized_keys = "${var.ssh_public_key}\n${tls_private_key.ssh.public_key_openssh}" 25 | user_data = base64encode(join("\n", list( 26 | "#!/usr/bin/env bash", 27 | "set -x", 28 | "version=\"${var.spectrum_scale_version}\"", 29 | "downloadUrl=\"${var.spectrum_scale_download_url}\"", 30 | "totalNsdNodePools=\"${var.total_nsd_node_pools}\"", 31 | "nsdNodesPerPool=\"${var.nsd_nodes_per_pool}\"", 32 | "nsdNodeCount=\"${(var.total_nsd_node_pools * var.nsd_nodes_per_pool)}\"", 33 | "nsdNodeHostnamePrefix=\"${var.nsd_node_hostname_prefix}\"", 34 | "clientNodeCount=\"${var.client_node_count}\"", 35 | "clientNodeHostnamePrefix=\"${var.client_node_hostname_prefix}\"", 36 | "blockSize=\"${var.spectrum_scale_block_size}\"", 37 | "dataReplica=\"${var.spectrum_scale_data_replica}\"", 38 | "metadataReplica=\"${var.spectrum_scale_metadata_replica}\"", 39 | "gpfsMountPoint=\"${var.spectrum_scale_gpfs_mount_point}\"", 40 | "sharedDataDiskCount=\"${(var.total_nsd_node_pools * var.block_volumes_per_pool)}\"", 41 | "blockVolumesPerPool=\"${var.block_volumes_per_pool}\"", 42 | "installerNode=\"${var.nsd_node_hostname_prefix}${var.installer_node}\"", 43 | "vcnFQDN=\"${local.vcn_domain_name}\"", 44 | "privateSubnetsFQDN=\"${local.storage_subnet_domain_name}\"", 45 | "privateBSubnetsFQDN=\"${local.filesystem_subnet_domain_name}\"", 46 | "cesNodeCount=\"${var.ces_node_count}\"", 47 | "cesNodeHostnamePrefix=\"${var.ces_node_hostname_prefix}\"", 48 | "mgmtGuiNodeCount=\"${var.mgmt_gui_node_count}\"", 49 | "mgmtGuiNodeHostnamePrefix=\"${var.mgmt_gui_node_hostname_prefix}\"", 50 | "privateProtocolSubnetFQDN=\"${local.protocol_subnet_domain_name}\"", 51 | file("${var.scripts_directory}/firewall.sh"), 52 | file("${var.scripts_directory}/set_env_variables.sh"), 53 | file("${var.scripts_directory}/update_resolv_conf.sh"), 54 | file("${var.scripts_directory}/configure_nic.sh"), 55 | file("${var.scripts_directory}/block_volume_discovery.sh"), 56 | file("${var.scripts_directory}/infra_tuning.sh"), 57 | file("${var.scripts_directory}/passwordless_ssh.sh"), 58 | file("${var.scripts_directory}/install_spectrum_scale.sh") 59 | ))) 60 | } 61 | 62 | timeouts { 63 | create = "120m" 64 | } 65 | 66 | } 67 | 68 | 69 | resource "null_resource" "deploy_ssh_keys_on_mgmt_gui_node" { 70 | depends_on = [ 71 | oci_core_instance.mgmt_gui_node, 72 | ] 73 | count = var.mgmt_gui_node_count 74 | triggers = { 75 | instance_ids = "oci_core_instance.mgmt_gui_node.*.id" 76 | } 77 | 78 | provisioner "file" { 79 | content = tls_private_key.ssh.private_key_pem 80 | destination = "/home/opc/.ssh/id_rsa" 81 | connection { 82 | agent = false 83 | timeout = "30m" 84 | host = element(oci_core_instance.mgmt_gui_node.*.private_ip, count.index) 85 | user = var.ssh_user 86 | private_key = tls_private_key.ssh.private_key_pem 87 | bastion_host = oci_core_instance.bastion[0].public_ip 88 | bastion_port = "22" 89 | bastion_user = var.ssh_user 90 | bastion_private_key = tls_private_key.ssh.private_key_pem 91 | } 92 | } 93 | 94 | provisioner "file" { 95 | content = tls_private_key.ssh.public_key_openssh 96 | destination = "/home/opc/.ssh/id_rsa.pub" 97 | connection { 98 | agent = false 99 | timeout = "30m" 100 | host = element(oci_core_instance.mgmt_gui_node.*.private_ip, count.index) 101 | user = var.ssh_user 102 | private_key = tls_private_key.ssh.private_key_pem 103 | bastion_host = oci_core_instance.bastion[0].public_ip 104 | bastion_port = "22" 105 | bastion_user = var.ssh_user 106 | bastion_private_key = tls_private_key.ssh.private_key_pem 107 | } 108 | } 109 | 110 | } 111 | 112 | /* Remote exec to deploy gpfs software/rpms on mgmt_gui nodes */ 113 | resource "null_resource" "deploy_gpfs_on_mgmt_gui_nodes" { 114 | depends_on = [ 115 | oci_core_instance.mgmt_gui_node, 116 | null_resource.notify_server_nodes_oci_cli_multi_attach_complete, 117 | ] 118 | count = var.mgmt_gui_node_count 119 | triggers = { 120 | instance_ids = "oci_core_instance.mgmt_gui_node.*.id" 121 | } 122 | 123 | provisioner "file" { 124 | source = "${var.scripts_directory}/" 125 | destination = "/tmp/" 126 | connection { 127 | agent = false 128 | timeout = "30m" 129 | host = element(oci_core_instance.mgmt_gui_node.*.private_ip, count.index) 130 | user = var.ssh_user 131 | private_key = tls_private_key.ssh.private_key_pem 132 | bastion_host = oci_core_instance.bastion[0].public_ip 133 | bastion_port = "22" 134 | bastion_user = var.ssh_user 135 | bastion_private_key = tls_private_key.ssh.private_key_pem 136 | } 137 | } 138 | 139 | provisioner "remote-exec" { 140 | connection { 141 | agent = false 142 | timeout = "30m" 143 | host = element(oci_core_instance.mgmt_gui_node.*.private_ip, count.index) 144 | user = var.ssh_user 145 | private_key = tls_private_key.ssh.private_key_pem 146 | bastion_host = oci_core_instance.bastion[0].public_ip 147 | bastion_port = "22" 148 | bastion_user = var.ssh_user 149 | bastion_private_key = tls_private_key.ssh.private_key_pem 150 | } 151 | inline = [ 152 | "set -x", 153 | "echo about to run /tmp/nodes-cloud-init-complete-status-check.sh", 154 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 155 | "sudo -s bash -c 'set -x && /tmp/nodes-cloud-init-complete-status-check.sh'", 156 | "sudo -s bash -c 'set -x && /tmp/deploy_spectrum_scale.sh'", 157 | ] 158 | } 159 | } 160 | 161 | 162 | 163 | 164 | /* Remote exec to configure mgmt_gui service on mgmt_gui node 1 */ 165 | resource "null_resource" "configure_mgmt_gui_service" { 166 | depends_on = [ 167 | oci_core_instance.mgmt_gui_node, 168 | null_resource.create_gpfs_cluster, 169 | null_resource.configure_ces_service 170 | ] 171 | count = var.mgmt_gui_node_count > 0 ? 1 : 0 172 | # 1 173 | triggers = { 174 | instance_ids = "oci_core_instance.mgmt_gui_node.*.id" 175 | } 176 | 177 | provisioner "remote-exec" { 178 | connection { 179 | agent = false 180 | timeout = "30m" 181 | host = element(oci_core_instance.mgmt_gui_node.*.private_ip, count.index) 182 | user = var.ssh_user 183 | private_key = tls_private_key.ssh.private_key_pem 184 | bastion_host = oci_core_instance.bastion[0].public_ip 185 | bastion_port = "22" 186 | bastion_user = var.ssh_user 187 | bastion_private_key = tls_private_key.ssh.private_key_pem 188 | } 189 | inline = [ 190 | "set -x", 191 | "sudo -s bash -c 'set -x && chmod 777 /tmp/*.sh'", 192 | "sudo su -l -c 'set -x && /tmp/configure_mgmt_gui.sh'", 193 | ] 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/datasources.tf: -------------------------------------------------------------------------------- 1 | # Gets a list of Availability Domains 2 | data "oci_identity_availability_domains" "ADs" { 3 | compartment_id = var.tenancy_ocid 4 | } 5 | 6 | 7 | 8 | data "oci_core_vcn" "vcn" { 9 | vcn_id = var.use_existing_vcn ? var.vcn_id : oci_core_vcn.vcn[0].id 10 | } 11 | 12 | data "oci_core_subnet" "storage_subnet" { 13 | subnet_id = var.use_existing_vcn ? var.storage_subnet_id : local.storage_subnet_id 14 | } 15 | 16 | data "oci_core_subnet" "fs_subnet" { 17 | subnet_id = var.use_existing_vcn ? var.fs_subnet_id : local.fs_subnet_id 18 | } 19 | 20 | data "oci_core_subnet" "protocol_subnet" { 21 | subnet_id = var.use_existing_vcn ? var.protocol_subnet_id : local.protocol_subnet_id 22 | } 23 | 24 | 25 | resource "tls_private_key" "ssh" { 26 | algorithm = "RSA" 27 | rsa_bits = "4096" 28 | } 29 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/multi_attach_storage.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /* 5 | Copy nsddevices file which is required to override default GPFS behavior to lookup NSD disk 6 | */ 7 | resource "null_resource" "copy_nsddevices_to_all_server_nodes" { 8 | depends_on = [oci_core_instance.nsd_node] 9 | count = var.total_nsd_node_pools * var.nsd_nodes_per_pool 10 | provisioner "file" { 11 | source = "${var.scripts_directory}/nsddevices" 12 | destination = "/tmp/nsddevices" 13 | connection { 14 | agent = false 15 | timeout = "30m" 16 | host = element(oci_core_instance.nsd_node.*.private_ip, count.index) 17 | user = var.ssh_user 18 | private_key = tls_private_key.ssh.private_key_pem 19 | bastion_host = oci_core_instance.bastion.*.public_ip[0] 20 | bastion_port = "22" 21 | bastion_user = var.ssh_user 22 | bastion_private_key = tls_private_key.ssh.private_key_pem 23 | } 24 | } 25 | } 26 | 27 | /* 28 | Logic to build OCI CLI commands to do multi-attach of BVol to compute instances 29 | */ 30 | locals { 31 | multi_attach_command_list = flatten( 32 | [for a in var.total_nsd_node_pools_list : 33 | [ 34 | [for b in var.nsd_nodes_per_pool_list : 35 | [ 36 | [for c in var.block_volumes_per_pool_list : 37 | [ 38 | "oci compute volume-attachment attach --type iscsi --is-shareable true --instance-id ${oci_core_instance.nsd_node[((a*var.nsd_nodes_per_pool)+b)].id} --volume-id ${oci_core_volume.shared_data_block_volume[((a*var.block_volumes_per_pool)+c)].id} --device ${var.volume_attach_device_mapping[(c)]} --config-file ~/.oci/config " 39 | ] if c < var.block_volumes_per_pool 40 | ] 41 | ] if b < var.nsd_nodes_per_pool 42 | ] 43 | ] if a < var.total_nsd_node_pools 44 | ] 45 | ) 46 | } 47 | 48 | locals { 49 | test_multi_attach_command_list = flatten( 50 | [for a in var.total_nsd_node_pools_list : 51 | [ 52 | [for b in var.nsd_nodes_per_pool_list : 53 | [ 54 | [for c in var.block_volumes_per_pool_list : 55 | [ 56 | "${((a*var.nsd_nodes_per_pool)+b)} ${((a*var.block_volumes_per_pool)+c)} ${(c)} " 57 | ] if c < var.block_volumes_per_pool 58 | ] 59 | ] if b < var.nsd_nodes_per_pool 60 | ] 61 | ] if a < var.total_nsd_node_pools 62 | ] 63 | ) 64 | } 65 | 66 | 67 | 68 | /* 69 | resource "null_resource" "test_multi_attach_shared_data_bv_to_nsd_nodes" { 70 | depends_on = [ 71 | oci_core_instance.nsd_node, 72 | oci_core_volume.shared_data_block_volume, 73 | null_resource.copy_nsddevices_to_all_server_nodes 74 | ] 75 | count = length(local.test_multi_attach_command_list) 76 | 77 | # 60-200 78 | provisioner "local-exec" { 79 | command = "echo ${local.test_multi_attach_command_list[count.index]} >> results.txt ; " 80 | } 81 | 82 | } 83 | */ 84 | 85 | 86 | /* 87 | Logic to run the OCI CLI commands to do multi-attach of BVol to compute instances 88 | */ 89 | /* 90 | resource "null_resource" "multi_attach_shared_data_bv_to_nsd_nodes" { 91 | depends_on = [ 92 | oci_core_instance.nsd_node, 93 | oci_core_volume.shared_data_block_volume, 94 | null_resource.copy_nsddevices_to_all_server_nodes, 95 | null_resource.test_multi_attach_shared_data_bv_to_nsd_nodes 96 | ] 97 | count = length(local.multi_attach_command_list) 98 | 99 | 100 | # 60-200 101 | provisioner "local-exec" { 102 | command = "delay=`shuf -i 5-30 -n 1` ; echo $delay ; sleep $delay ; " 103 | } 104 | # ${local.multi_attach_command_list[count.index]} ; 105 | } 106 | */ 107 | 108 | 109 | 110 | resource "oci_core_volume_attachment" "shared_data_blockvolume_attach" { 111 | attachment_type = "iscsi" 112 | count = length(local.test_multi_attach_command_list) 113 | 114 | instance_id = element(oci_core_instance.nsd_node.*.id, element(split(" ", local.test_multi_attach_command_list[count.index]),0) ,) 115 | volume_id = element(oci_core_volume.shared_data_block_volume.*.id, ( element(split(" ", local.test_multi_attach_command_list[count.index]),1) ) ) 116 | is_shareable = true 117 | 118 | device = var.volume_attach_device_mapping[( element(split(" ", local.test_multi_attach_command_list[count.index]),2) )] 119 | 120 | 121 | provisioner "remote-exec" { 122 | connection { 123 | agent = false 124 | timeout = "30m" 125 | host = element(oci_core_instance.nsd_node.*.private_ip, ( element(split(" ", local.test_multi_attach_command_list[count.index]),0) ) ) 126 | 127 | user = var.ssh_user 128 | private_key = tls_private_key.ssh.private_key_pem 129 | bastion_host = oci_core_instance.bastion[0].public_ip 130 | bastion_port = "22" 131 | bastion_user = var.ssh_user 132 | bastion_private_key = tls_private_key.ssh.private_key_pem 133 | 134 | } 135 | 136 | inline = [ 137 | "sudo -s bash -c 'set -x && iscsiadm -m node -o new -T ${self.iqn} -p ${self.ipv4}:${self.port}'", 138 | "sudo -s bash -c 'set -x && iscsiadm -m node -o update -T ${self.iqn} -n node.startup -v automatic '", 139 | "sudo -s bash -c 'set -x && iscsiadm -m node -T ${self.iqn} -p ${self.ipv4}:${self.port} -l '", 140 | "delay=`shuf -i 5-30 -n 1` ; echo $delay ; sleep $delay ;" 141 | ] 142 | } 143 | } 144 | 145 | 146 | 147 | 148 | /* 149 | Notify NSD server nodes that multi-attach is complete, so NSD server nodes can continue with their rest of the instance setup logic in cloud-init. 150 | */ 151 | resource "null_resource" "notify_server_nodes_oci_cli_multi_attach_complete" { 152 | depends_on = [ 153 | oci_core_volume_attachment.shared_data_blockvolume_attach, 154 | null_resource.copy_nsddevices_to_all_server_nodes] 155 | count = var.total_nsd_node_pools * var.nsd_nodes_per_pool 156 | provisioner "remote-exec" { 157 | connection { 158 | agent = false 159 | timeout = "30m" 160 | host = element(oci_core_instance.nsd_node.*.private_ip, count.index) 161 | user = var.ssh_user 162 | private_key = tls_private_key.ssh.private_key_pem 163 | bastion_host = oci_core_instance.bastion.*.public_ip[0] 164 | bastion_port = "22" 165 | bastion_user = var.ssh_user 166 | bastion_private_key = tls_private_key.ssh.private_key_pem 167 | } 168 | inline = [ 169 | "set -x", 170 | "sudo touch /tmp/multi-attach.complete", 171 | ] 172 | } 173 | } 174 | 175 | 176 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/network.tf: -------------------------------------------------------------------------------- 1 | /* 2 | All network resources for this template 3 | */ 4 | 5 | resource "oci_core_vcn" "vcn" { 6 | count = var.use_existing_vcn ? 0 : 1 7 | cidr_block = var.vpc_cidr 8 | compartment_id = var.compartment_ocid 9 | display_name = "gpfs" 10 | dns_label = "gpfs" 11 | } 12 | 13 | resource "oci_core_internet_gateway" "internet_gateway" { 14 | count = var.use_existing_vcn ? 0 : 1 15 | compartment_id = var.compartment_ocid 16 | display_name = "internet_gateway" 17 | vcn_id = oci_core_vcn.vcn[0].id 18 | } 19 | 20 | resource "oci_core_route_table" "pubic_route_table" { 21 | count = var.use_existing_vcn ? 0 : 1 22 | compartment_id = var.compartment_ocid 23 | vcn_id = oci_core_vcn.vcn[0].id 24 | display_name = "RouteTableForComplete" 25 | route_rules { 26 | cidr_block = "0.0.0.0/0" 27 | network_entity_id = oci_core_internet_gateway.internet_gateway[0].id 28 | } 29 | } 30 | 31 | 32 | resource "oci_core_nat_gateway" "nat_gateway" { 33 | count = var.use_existing_vcn ? 0 : 1 34 | compartment_id = var.compartment_ocid 35 | vcn_id = oci_core_vcn.vcn[0].id 36 | display_name = "nat_gateway" 37 | } 38 | 39 | 40 | resource "oci_core_route_table" "private_route_table" { 41 | count = var.use_existing_vcn ? 0 : 1 42 | compartment_id = var.compartment_ocid 43 | vcn_id = oci_core_vcn.vcn[0].id 44 | display_name = "private_route_tableForComplete" 45 | route_rules { 46 | destination = "0.0.0.0/0" 47 | network_entity_id = oci_core_nat_gateway.nat_gateway[0].id 48 | } 49 | } 50 | 51 | resource "oci_core_security_list" "public_security_list" { 52 | count = var.use_existing_vcn ? 0 : 1 53 | compartment_id = var.compartment_ocid 54 | display_name = "Public Subnet" 55 | vcn_id = oci_core_vcn.vcn[0].id 56 | egress_security_rules { 57 | destination = "0.0.0.0/0" 58 | protocol = "6" 59 | } 60 | ingress_security_rules { 61 | tcp_options { 62 | max = 22 63 | min = 22 64 | } 65 | protocol = "6" 66 | source = "0.0.0.0/0" 67 | } 68 | ingress_security_rules { 69 | tcp_options { 70 | max = 3389 71 | min = 3389 72 | } 73 | protocol = "6" 74 | source = "0.0.0.0/0" 75 | } 76 | } 77 | 78 | # https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewall.htm 79 | resource "oci_core_security_list" "private_security_list" { 80 | count = var.use_existing_vcn ? 0 : 1 81 | compartment_id = var.compartment_ocid 82 | display_name = "Private" 83 | vcn_id = oci_core_vcn.vcn[0].id 84 | 85 | egress_security_rules { 86 | destination = "0.0.0.0/0" 87 | protocol = "all" 88 | } 89 | egress_security_rules { 90 | protocol = "all" 91 | destination = var.vpc_cidr 92 | } 93 | # for Mgmt GUI: https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewallforgui.htm 94 | ingress_security_rules { 95 | tcp_options { 96 | max = 443 97 | min = 443 98 | } 99 | protocol = "6" 100 | source = var.vpc_cidr 101 | } 102 | ingress_security_rules { 103 | tcp_options { 104 | max = 22 105 | min = 22 106 | } 107 | protocol = "6" 108 | source = var.vpc_cidr 109 | } 110 | ingress_security_rules { 111 | tcp_options { 112 | max = 80 113 | min = 80 114 | } 115 | protocol = "6" 116 | source = var.vpc_cidr 117 | } 118 | ingress_security_rules { 119 | tcp_options { 120 | max = 443 121 | min = 443 122 | } 123 | protocol = "6" 124 | source = var.vpc_cidr 125 | } 126 | # for Object Storage on CES node: https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewallforprotaccess.htm 127 | ingress_security_rules { 128 | tcp_options { 129 | max = 8080 130 | min = 8080 131 | } 132 | protocol = "6" 133 | source = var.vpc_cidr 134 | } 135 | # for SMB on CES node: https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewallforprotaccess.htm 136 | ingress_security_rules { 137 | tcp_options { 138 | max = 445 139 | min = 445 140 | } 141 | protocol = "6" 142 | source = var.vpc_cidr 143 | } 144 | ingress_security_rules { 145 | tcp_options { 146 | max = 4379 147 | min = 4379 148 | } 149 | protocol = "6" 150 | source = var.vpc_cidr 151 | } 152 | 153 | # for NFSV4&NFSV3 on CES node: https://www.ibm.com/support/knowledgecenter/en/STXKQY_5.0.3/com.ibm.spectrum.scale.v5r03.doc/bl1adv_firewallforprotaccess.htm 154 | ingress_security_rules { 155 | tcp_options { 156 | max = 2049 157 | min = 2049 158 | } 159 | protocol = "6" 160 | source = var.vpc_cidr 161 | } 162 | ingress_security_rules { 163 | tcp_options { 164 | max = 111 165 | min = 111 166 | } 167 | protocol = "6" 168 | source = var.vpc_cidr 169 | } 170 | ingress_security_rules { 171 | tcp_options { 172 | max = 32765 173 | min = 32765 174 | } 175 | protocol = "6" 176 | source = var.vpc_cidr 177 | } 178 | ingress_security_rules { 179 | tcp_options { 180 | max = 32767 181 | min = 32767 182 | } 183 | protocol = "6" 184 | source = var.vpc_cidr 185 | } 186 | ingress_security_rules { 187 | tcp_options { 188 | max = 32768 189 | min = 32768 190 | } 191 | protocol = "6" 192 | source = var.vpc_cidr 193 | } 194 | ingress_security_rules { 195 | tcp_options { 196 | max = 32769 197 | min = 32769 198 | } 199 | protocol = "6" 200 | source = var.vpc_cidr 201 | } 202 | ingress_security_rules { 203 | udp_options { 204 | max = 2049 205 | min = 2049 206 | } 207 | protocol = "17" 208 | source = var.vpc_cidr 209 | } 210 | ingress_security_rules { 211 | udp_options { 212 | max = 111 213 | min = 111 214 | } 215 | protocol = "17" 216 | source = var.vpc_cidr 217 | } 218 | ingress_security_rules { 219 | udp_options { 220 | max = 32765 221 | min = 32765 222 | } 223 | protocol = "17" 224 | source = var.vpc_cidr 225 | } 226 | ingress_security_rules { 227 | udp_options { 228 | max = 32767 229 | min = 32767 230 | } 231 | protocol = "17" 232 | source = var.vpc_cidr 233 | } 234 | ingress_security_rules { 235 | udp_options { 236 | max = 32768 237 | min = 32768 238 | } 239 | protocol = "17" 240 | source = var.vpc_cidr 241 | } 242 | ingress_security_rules { 243 | udp_options { 244 | max = 32769 245 | min = 32769 246 | } 247 | protocol = "17" 248 | source = var.vpc_cidr 249 | } 250 | 251 | ingress_security_rules { 252 | protocol = "All" 253 | source = var.vpc_cidr 254 | } 255 | } 256 | 257 | 258 | # Regional subnet - public 259 | resource "oci_core_subnet" "public" { 260 | count = var.use_existing_vcn ? 0 : 1 261 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index) 262 | #display_name = "${local.cluster_name}_public" 263 | display_name = "Public-Subnet" 264 | compartment_id = var.compartment_ocid 265 | vcn_id = oci_core_vcn.vcn[0].id 266 | route_table_id = oci_core_route_table.pubic_route_table[0].id 267 | security_list_ids = [oci_core_security_list.public_security_list[0].id] 268 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 269 | dns_label = "public" 270 | } 271 | 272 | 273 | # Regional subnet - private 274 | resource "oci_core_subnet" "storage" { 275 | count = var.use_existing_vcn ? 0 : 1 276 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index+3) 277 | display_name = "Private-SpectrumScale" 278 | compartment_id = var.compartment_ocid 279 | vcn_id = oci_core_vcn.vcn[0].id 280 | route_table_id = oci_core_route_table.private_route_table[0].id 281 | security_list_ids = [oci_core_security_list.private_security_list[0].id] 282 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 283 | prohibit_public_ip_on_vnic = "true" 284 | dns_label = "storage" 285 | } 286 | 287 | # Regional subnet - private B 288 | resource "oci_core_subnet" "fs" { 289 | count = var.use_existing_vcn ? 0 : 1 290 | #(local.dual_nics ? 1 : 0) 291 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index+6) 292 | display_name = "Private-FS-Subnet" 293 | compartment_id = var.compartment_ocid 294 | vcn_id = oci_core_vcn.vcn[0].id 295 | route_table_id = oci_core_route_table.private_route_table[0].id 296 | security_list_ids = [oci_core_security_list.private_security_list[0].id] 297 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 298 | prohibit_public_ip_on_vnic = "true" 299 | dns_label = "fs" 300 | } 301 | 302 | # Regional subnet - private for CES/TCT/Protocol nodes. 303 | resource "oci_core_subnet" "protocol_subnet" { 304 | count = var.use_existing_vcn ? 0 : 1 305 | #(local.dual_nics ? 1 : 0) 306 | cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index+9) 307 | display_name = "privateprotocol_${count.index}" 308 | compartment_id = var.compartment_ocid 309 | vcn_id = oci_core_vcn.vcn[0].id 310 | route_table_id = oci_core_route_table.private_route_table[0].id 311 | security_list_ids = [oci_core_security_list.private_security_list[0].id] 312 | dhcp_options_id = oci_core_vcn.vcn[0].default_dhcp_options_id 313 | prohibit_public_ip_on_vnic = "true" 314 | dns_label = "privprotocol${count.index}" 315 | } 316 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | output "SSH-login" { 4 | value = <$OUT 20 | sensors = { 21 | # NFS Ganesha statistics 22 | name = "NFSIO" 23 | period = 1 24 | restrict = "cesNodes" 25 | type = "Generic" 26 | }, { 27 | name = "SMBStats" 28 | period = 1 29 | restrict = "cesNodes" 30 | type = "Generic" 31 | }, { 32 | name = "SMBGlobalStats" 33 | period = 1 34 | restrict = "cesNodes" 35 | type = "Generic" 36 | }, { 37 | name = "CTDBStats" 38 | period = 1 39 | restrict = "cesNodes" 40 | type = "Generic" 41 | }, { 42 | name = "CTDBDBStats" 43 | period = 1 44 | restrict = "cesNodes" 45 | type = "Generic" 46 | } 47 | EOF 48 | 49 | # Add the sensors to existing sensors. 50 | mmperfmon config add --sensors $OUT 51 | 52 | 53 | # Happens automatically 54 | # Step 7. Federate the pm collectors. Modify following lines in /opt/IBM/zimon/ZIMonCollector.cfg file on both pm collector nodes. 55 | #[root@ss-mgmt-gui-1 ~] 56 | # cat /opt/IBM/zimon/ZIMonCollector.cfg 57 | #... 58 | #... 59 | #peers = { 60 | # host = "ss-mgmt-gui-1.privateb0.gpfs.oraclevcn.com" 61 | # port = "9085" 62 | #}, { 63 | # host = "ss-mgmt-gui-2.privateb0.gpfs.oraclevcn.com" 64 | # port = "9085" 65 | #} 66 | #... 67 | 68 | 69 | # Step 8. Restart the pm collector and pm sensors services. 70 | mmdsh -N $allMgmtGuiNodes "systemctl restart pmcollector" 71 | mmdsh -N all "systemctl restart pmsensors" 72 | 73 | # Step 9. Enable and start the gpfsgui services. 74 | # Starting version 5.0.5.0 or may be 5.0.4.0, this change is required. 75 | mmdsh -N $allMgmtGuiNodes 'sed -i "s|java -XX|java -Xmx2048m -XX|g" /usr/lib/systemd/system/gpfsgui.service' 76 | mmdsh -N $allMgmtGuiNodes "systemctl stop gpfsgui.service" 77 | mmdsh -N $allMgmtGuiNodes "systemctl start gpfsgui.service" 78 | mmdsh -N $allMgmtGuiNodes "systemctl enable gpfsgui.service" 79 | 80 | 81 | # Step 10. Create the GUI admin account. 82 | mmdsh -N $allMgmtGuiNodes "/usr/lpp/mmfs/gui/cli/initgui" 83 | 84 | # Run on [root@ss-mgmt-gui-1 ~] 85 | echo -e "passw0rd\npassw0rd" | /usr/lpp/mmfs/gui/cli/mkuser admin -g SecurityAdmin 86 | # EFSSG1007A Enter password for User : 87 | # EFSSG0225I Repeat the password: 88 | # EFSSG0019I The user admin has been successfully created. 89 | # EFSSG1000I The command completed successfully. 90 | 91 | # Step 11. Start GPFS deamon. 92 | mmstartup -N $allMgmtGuiNodes 93 | 94 | while [ `mmgetstate -a | grep "$mgmtGuiNodeHostnamePrefix" | grep "active" | wc -l` -lt $((mgmtGuiNodeCount)) ] ; do echo "waiting for mgmt gui nodes of cluster to start ..." ; sleep 10s; done; 95 | 96 | 97 | # To see master GUI node. You can run this on all MGMT GUI nodes, if you have more than 1 98 | # One of them will show "Master GUI Node" , other will show just "GUI Node" 99 | /usr/lpp/mmfs/gui/cli/lsnode 100 | 101 | # From your local machine, create an ssh tunnel using bastion host as the intermediatory to reach the GUI on the GUI Mgmt node which is in private subnet. 102 | 103 | # ssh -i ~/.ssh/oci -N -L localhost:11443:10.0.3.5:443 opc@129.146.189.209 104 | 105 | # bastion = 129.146.189.209 106 | # Remote node (private) = 10.0.6.5 107 | # Assuming a GUI running on default 443 port on 10.0.6.5 108 | 109 | # Using above command, I am connecting my MacBook (localhost) on port 11443 to create a tunnel to 10.0.6.5:443 and its via the bastion host. Hence we do ssh to the bastion host above. 110 | 111 | # Open browser locally and use https (not http): https://localhost:11443 112 | 113 | # Generate file using any client node and see the throughput using GUI. 114 | # dd if=/dev/zero of=/gpfs/fs1/test.file bs=1024k count=100000 115 | # dd if=/dev/zero of=/gpfs/fs1/test.file bs=1024k count=1000000 116 | 117 | 118 | #Other commands - not required for deployment 119 | ## mmperfmon query compareNodes cpu 120 | ## systemctl status gpfsgui 121 | ## cd /opt/IBM/zimon 122 | ## more ZIMonCollector.cfg 123 | ## echo "get metrics mem_active, cpu_idle, gpfs_ns_read_ops, last 10 bucket_size 1" | ./zc 127.0.0.1 124 | ## more /etc/gss/mech.d/gssproxy.conf 125 | ## more /etc/gssproxy/gssproxy.conf 126 | ## /usr/lpp/mmfs/gui/cli/lsnode 127 | 128 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/configure_nic.sh: -------------------------------------------------------------------------------- 1 | MDATA_VNIC_URL="http://169.254.169.254/opc/v1/vnics/" 2 | 3 | function configure_vnics { 4 | # Configure 2nd vNIC 5 | scriptsource="https://raw.githubusercontent.com/oracle/terraform-examples/master/examples/oci/connect_vcns_using_multiple_vnics/scripts/secondary_vnic_all_configure.sh" 6 | vnicscript=/root/secondary_vnic_all_configure.sh 7 | curl -s $scriptsource > $vnicscript 8 | chmod +x $vnicscript 9 | cat > /etc/systemd/system/secondnic.service << EOF 10 | [Unit] 11 | Description=Script to configure a secondary vNIC 12 | 13 | [Service] 14 | Type=oneshot 15 | ExecStart=$vnicscript -c 16 | ExecStop=$vnicscript -d 17 | RemainAfterExit=yes 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | 22 | EOF 23 | 24 | systemctl enable secondnic.service 25 | systemctl start secondnic.service 26 | sleep 10s 27 | vnic_cnt=`/root/secondary_vnic_all_configure.sh | grep "ocid1.vnic." | grep " UP " | wc -l` ; 28 | RC=1 29 | interface="" 30 | while ( [ $vnic_cnt -le 1 ] || [ $RC -ne 0 ] ) 31 | do 32 | systemctl restart secondnic.service 33 | echo "sleep 10s" 34 | sleep 10s 35 | privateIp=`curl -s http://169.254.169.254/opc/v1/vnics/ | jq '.[1].privateIp ' | sed 's/"//g' ` ; echo $privateIp 36 | interface=`ip addr | grep -B2 $privateIp | grep "BROADCAST" | gawk -F ":" ' { print $2 } ' | sed -e 's/^[ \t]*//'` ; echo $interface 37 | if [ -z $interface ]; then 38 | RC=1 39 | else 40 | RC=0 41 | fi 42 | vnic_cnt=`/root/secondary_vnic_all_configure.sh | grep "ocid1.vnic." | grep " UP " | wc -l` ; 43 | done 44 | 45 | } 46 | 47 | function configure_2nd_VNIC { 48 | 49 | configure_vnics 50 | # if 1 or 2 VNIC. 51 | vnic_count=`curl -s $MDATA_VNIC_URL | jq '. | length'` 52 | 53 | if [ $vnic_count -gt 1 ] ; then 54 | echo "2 VNIC setup" 55 | 56 | privateIp=`curl -s $MDATA_VNIC_URL | jq '.[1].privateIp ' | sed 's/"//g' ` ; echo $privateIp 57 | interface=`ip addr | grep -B2 $privateIp | grep "BROADCAST" | gawk -F ":" ' { print $2 } ' | sed -e 's/^[ \t]*//'` ; echo $interface 58 | 59 | if [ "$intel_node" = "true" ]; then 60 | if [ "$hpc_node" = "true" ]; then 61 | echo "don't tune on hpc shape" 62 | else 63 | tune_interface 64 | fi 65 | fi 66 | 67 | # ensure its not empty 68 | test=`nslookup $privateIp | grep -q "name = "` 69 | while [ $? -ne 0 ]; 70 | do 71 | echo "Waiting for nslookup..." 72 | sleep 10s 73 | test=`nslookup $privateIp | grep -q "name = "` 74 | done 75 | 76 | secondNicFQDNHostname=`nslookup $privateIp | grep "name = " | gawk -F"=" '{ print $2 }' | sed "s|^ ||g" | sed "s|\.$||g"` 77 | thisFQDN=$secondNicFQDNHostname 78 | thisHost=${thisFQDN%%.*} 79 | secondNICDomainName=${thisFQDN#*.*} 80 | echo $secondNICDomainName 81 | primaryNICHostname="`hostname`" 82 | else 83 | echo "Server nodes with 1 physical NIC - get hostname for 1st NIC..." 84 | set_env_variables 85 | fi 86 | } 87 | 88 | function tune_interface { 89 | ethtool -G $interface rx 2047 tx 2047 rx-jumbo 8191 90 | ethtool -L $interface combined 74 91 | echo "ethtool -G $interface rx 2047 tx 2047 rx-jumbo 8191" >> /etc/rc.local 92 | echo "ethtool -L $interface combined 74" >> /etc/rc.local 93 | chmod +x /etc/rc.local 94 | } 95 | 96 | function set_env_variables { 97 | thisFQDN=`hostname --fqdn` 98 | thisHost=${thisFQDN%%.*} 99 | } 100 | 101 | coreIdCount=`grep "^core id" /proc/cpuinfo | sort -u | wc -l` ; 102 | socketCount=`echo $(($(grep "^physical id" /proc/cpuinfo | awk '{print $4}' | sort -un | tail -1)+1))` ; 103 | 104 | 105 | # configure 1st NIC 106 | privateIp=`curl -s $MDATA_VNIC_URL | jq '.[0].privateIp ' | sed 's/"//g' ` ; echo $privateIp 107 | interface=`ip addr | grep -B2 $privateIp | grep "BROADCAST" | gawk -F ":" ' { print $2 } ' | sed -e 's/^[ \t]*//'` ; echo $interface 108 | 109 | hpc_node=false 110 | intel_node=true 111 | lscpu | grep "Vendor ID:" | grep "AuthenticAMD" 112 | if [ $? -eq 0 ]; then 113 | echo "do nothing - AMD" 114 | intel_node=false 115 | else 116 | if [ $((socketCount*coreIdCount)) -eq 36 ]; then 117 | echo "skip for hpc" 118 | hpc_node=true 119 | else 120 | tune_interface 121 | fi 122 | fi 123 | 124 | # 2nd NIC 125 | echo `hostname` | grep -q "$clientNodeHostnamePrefix\|$mgmtGuiNodeHostnamePrefix" 126 | if [ $? -eq 0 ] ; then 127 | set_env_variables 128 | else 129 | echo `hostname` | grep -q "$cesNodeHostnamePrefix" 130 | if [ $? -eq 0 ] ; then 131 | configure_2nd_VNIC 132 | else 133 | if [ "$privateSubnetsFQDN" = "$privateBSubnetsFQDN" ]; then 134 | set_env_variables 135 | else 136 | configure_2nd_VNIC 137 | fi 138 | fi 139 | fi 140 | 141 | # required 142 | echo "thisFQDN=\"$thisFQDN\"" >> /tmp/gpfs_env_variables.sh 143 | echo "thisHost=\"$thisHost\"" >> /tmp/gpfs_env_variables.sh 144 | 145 | echo $thisHost | grep -q "$cesNodeHostnamePrefix" 146 | if [ $? -eq 0 ] ; then 147 | 148 | # TODO: fix it This assume 3rd in the list is VIP IP and 2nd will be for privateb subnet 149 | privateVipIp=`curl -s $MDATA_VNIC_URL | jq '.[2].privateIp ' | sed 's/"//g' ` ; 150 | echo $privateVipIp | grep "\." ; 151 | while [ $? -ne 0 ]; 152 | do 153 | sleep 10s 154 | echo "Waiting for IP of VNIC to get configured..." 155 | privateVipIp=`curl -s $MDATA_VNIC_URL | jq '.[2].privateIp ' | sed 's/"//g' ` ; 156 | echo $privateVipIp | grep "\." ; 157 | done 158 | echo "$privateVipIp" >> /tmp/ces_vip_ips 159 | fi 160 | 161 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/create_spectrum_scale_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | source /tmp/gpfs_env_variables.sh 6 | 7 | # Only on installerNode 8 | echo "$thisHost" | grep -q -w $installerNode 9 | if [ $? -eq 0 ] ; then 10 | rm /root/node.stanza 11 | qcount=1 12 | for hname in `cat /tmp/allnodehosts` ; do 13 | echo "$hname" | grep -q $nsdNodeHostnamePrefix 14 | if [ $? -eq 0 ] ; then 15 | if [ $qcount -le 5 ]; then 16 | echo "${hname}:quorum-manager" >> /root/node.stanza 17 | qcount=$((qcount+1)) 18 | else 19 | echo "${hname}" >> /root/node.stanza 20 | fi 21 | fi 22 | echo "$hname" | grep -q $clientNodeHostnamePrefix 23 | if [ $? -eq 0 ] ; then 24 | echo "${hname}" >> /root/node.stanza 25 | fi 26 | done 27 | cat /root/node.stanza 28 | 29 | mmcrcluster -N node.stanza -r /usr/bin/ssh -R /usr/bin/scp -C ss-storage-cluster -A 30 | sleep 30s 31 | 32 | diskArray=(b c d e f g h i j k l m n o p q r s t u v w x y z aa ab ac ad ae af ag) 33 | 34 | command="mmchlicense server --accept -N " 35 | for node in `seq 1 $nsdNodeCount`; 36 | do 37 | echo $node 38 | if [ $node -eq $nsdNodeCount ]; then 39 | # no comma at the end 40 | command="${command}${node}" 41 | else 42 | command="${command}${node}," 43 | fi 44 | done 45 | echo $command 46 | $command 47 | 48 | sleep 30s 49 | 50 | if [ $clientNodeCount -gt 0 ]; then 51 | startIndex=$((nsdNodeCount+1)) 52 | endIndex=$((nsdNodeCount+clientNodeCount)) 53 | command="mmchlicense client --accept -N " 54 | for node in `seq $startIndex $endIndex`; 55 | do 56 | echo $node 57 | if [ $node -eq $endIndex ]; then 58 | # no comma 59 | command="${command}${node}" 60 | else 61 | command="${command}${node}," 62 | fi 63 | done 64 | echo $command 65 | $command 66 | else 67 | echo "No client nodes to configure" 68 | fi 69 | 70 | sleep 30s 71 | 72 | rm /tmp/nsd.stanza.sv* 73 | for poolIndex in `seq 1 $totalNsdNodePools`; 74 | do 75 | 76 | 77 | setFileName="/tmp/nsd.stanza.sv${nsdNodesPerPool}.set${poolIndex}" 78 | rm $setFileName 79 | for i in `seq 1 $blockVolumesPerPool`; 80 | do 81 | if [ $((i % 2)) -eq 0 ]; then 82 | primaryServer="${nsdNodeHostnamePrefix}$((((poolIndex-1)*nsdNodesPerPool)+2))" 83 | secondaryServer="${nsdNodeHostnamePrefix}$((((poolIndex-1)*nsdNodesPerPool)+1))" 84 | else 85 | primaryServer="${nsdNodeHostnamePrefix}$((((poolIndex-1)*nsdNodesPerPool)+1))" 86 | secondaryServer="${nsdNodeHostnamePrefix}$((((poolIndex-1)*nsdNodesPerPool)+2))" 87 | fi 88 | 89 | if ([ $metadataReplica -gt 1 ] || [ $dataReplica -gt 1 ]); then 90 | echo $i 91 | if [ $((i % 2)) -eq 0 ]; then 92 | failureGroup=102 93 | else 94 | failureGroup=101 95 | fi 96 | else 97 | failureGroup=100 98 | fi 99 | 100 | setFileName="/tmp/nsd.stanza.sv${nsdNodesPerPool}.set${poolIndex}" 101 | echo " " >> $setFileName 102 | echo "%nsd: nsd=nsd$((((poolIndex-1)*blockVolumesPerPool)+i))" >> $setFileName 103 | echo "device=/dev/oracleoci/oraclevd${diskArray[(($i-1))]}" >> $setFileName 104 | echo "servers=$primaryServer,$secondaryServer" >> $setFileName 105 | echo "usage=dataAndMetadata" >> $setFileName 106 | echo "pool=system" >> $setFileName 107 | echo "failureGroup=$failureGroup" >> $setFileName 108 | done 109 | 110 | done 111 | 112 | #Create the NSDs from all NSD stanza files. 113 | for poolIndex in `seq 1 $totalNsdNodePools`; 114 | do 115 | echo "mmcrnsd -F /tmp/nsd.stanza.sv${nsdNodesPerPool}.set${poolIndex} " 116 | mmcrnsd -F /tmp/nsd.stanza.sv${nsdNodesPerPool}.set${poolIndex} 117 | sleep 15s 118 | done 119 | 120 | mmchconfig maxblocksize=16M,maxMBpS=6250,numaMemoryInterleave=yes,tscCmdPortRange=60000-61000,workerThreads=1024 121 | # Change these values based on VM/BM shape. The below are for BM shape. 122 | # mmchconfig pagepool=128G,maxFilesToCache=5M -N nsdNodes 123 | # mmchconfig pagepool=64G,maxFilesToCache=1M -N clientNodes 124 | 125 | 126 | 127 | mmstartup -N nsdNodes 128 | while [ `mmgetstate -a | grep "active" | wc -l` -ne $((nsdNodeCount)) ] ; do echo "waiting for server nodes of cluster to start ..." ; sleep 10s; done; 129 | 130 | mmumount fs1 -a 131 | sleep 15s 132 | mmmount fs1 -a 133 | sleep 15s 134 | 135 | if [ $clientNodeCount -gt 0 ]; then 136 | mmstartup -N clientNodes 137 | while [ `mmgetstate -a | grep "active" | wc -l` -ne $((nsdNodeCount + clientNodeCount)) ] ; do echo "waiting for client nodes of cluster to start ..." ; sleep 10s; done; 138 | fi 139 | 140 | # Consolidate, since both failure groups needs to be in the same file, for the command the work. 141 | rm /tmp/nsd.stanza.sv${nsdNodesPerPool}.consolidated 142 | for poolIndex in `seq 1 $totalNsdNodePools`; 143 | do 144 | echo "consolidating the files... /tmp/nsd.stanza.sv${nsdNodesPerPool}.set${poolIndex} into /tmp/nsd.stanza.sv${nsdNodesPerPool}.consolidated " 145 | cat /tmp/nsd.stanza.sv${nsdNodesPerPool}.set${poolIndex} >> /tmp/nsd.stanza.sv${nsdNodesPerPool}.consolidated 146 | sleep 15s 147 | done 148 | 149 | # Create file system 150 | mmcrfs fs1 -F /tmp/nsd.stanza.sv${nsdNodesPerPool}.consolidated -B $blockSize -m $metadataReplica -M 2 -r $dataReplica -R 2 151 | sleep 60s 152 | 153 | mmrestripefs fs1 -b 154 | 155 | mmlsnsd 156 | mmlsfs fs1 157 | mmlsdisk fs1 -L 158 | 159 | mmmount fs1 -a 160 | sleep 15s 161 | df -h 162 | 163 | ### CES Nodes 164 | ## mmaddnode needs to be ran on a node which is already part of the cluster 165 | # Step. Assign the quorum role to one of protocol node and the manager role to both nodes. 166 | for node in `cat /tmp/cesnodehosts` ; do 167 | mmaddnode -N $node 168 | mmchlicense server --accept -N $node 169 | mmchnode --manager -N $node 170 | mmstartup -N $node 171 | done 172 | while [ `mmgetstate -a | grep "$cesNodeHostnamePrefix" | grep "active" | wc -l` -lt $((cesNodeCount)) ] ; do echo "waiting for ces nodes of cluster to start ..." ; sleep 10s; done; 173 | 174 | 175 | # To ensure pmsensors is installed on all nodes. 176 | # mmdsh -N all "rpm -qa | grep gpfs.gss.pmsensors" 177 | # To ensure pmcollector is installed on only GUI mgmt nodes. 178 | # mmdsh -N all "rpm -qa | grep gpfs.gss.pmcollector" 179 | 180 | for node in `cat /tmp/mgmtguinodehosts` ; do 181 | mmaddnode -N $node 182 | mmchlicense client --accept -N $node 183 | # recommended - 32G, if node has memory 184 | #mmchconfig pagepool=32G -N $node 185 | mmchconfig pagepool=10G -N $node 186 | done 187 | 188 | if [ $clientNodeCount -eq 0 ]; then 189 | mmauth genkey new 190 | mmauth update . -l AUTHONLY 191 | cp /var/mmfs/ssl/id_rsa.pub /home/opc/owningCluster_id_rsa.pub 192 | chown opc:opc /home/opc/owningCluster_id_rsa.pub 193 | 194 | # run the below on ss-server-1 node, after the ss-compute-only cluster is provisioned and you have the value of accessingClusterName and accessingCluster_id_rsa.pub file copied to ss-server-1 server. example: 195 | # filesystemName=fs1 196 | # accessingClusterName=ss-compute-only.fs.gpfs.oraclevcn.com 197 | # accessingClusterAuthPublicKeyFilePath=/home/opc/accessingCluster_id_rsa.pub 198 | # mmauth add $accessingClusterName -k $accessingClusterAuthPublicKeyFilePath 199 | # mmauth grant $accessingClusterName -f ${filesystemName} 200 | fi 201 | 202 | 203 | fi 204 | 205 | exit 0; 206 | 207 | 208 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/deploy_spectrum_scale.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | source /tmp/gpfs_env_variables.sh 6 | 7 | 8 | # Build the GPFS potability layer. 9 | /usr/lpp/mmfs/bin/mmbuildgpl 10 | # have seen the above fail sometimes, hence the below loop 11 | while [ $? -ne 0 ]; do 12 | sleep 10s; 13 | /usr/lpp/mmfs/bin/mmbuildgpl 14 | done; 15 | 16 | 17 | # Up date the PATH environmental variable. 18 | echo -e '\nexport PATH=/usr/lpp/mmfs/bin:$PATH' >> ~/.bash_profile 19 | source ~/.bash_profile 20 | 21 | exit 0; 22 | 23 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/firewall.sh: -------------------------------------------------------------------------------- 1 | echo "Turning off the Firewall..." 2 | service firewalld stop 3 | chkconfig firewalld off 4 | 5 | echo `hostname` | grep -q "$clientNodeHostnamePrefix\|$mgmtGuiNodeHostnamePrefix" 6 | if [ $? -eq 0 ] ; then 7 | echo "continue, no sleep required" 8 | else 9 | coreIdCount=`grep "^core id" /proc/cpuinfo | sort -u | wc -l` 10 | socketCount=`echo $(($(grep "^physical id" /proc/cpuinfo | awk '{print $4}' | sort -un | tail -1)+1))` 11 | if [ $((socketCount*coreIdCount)) -eq 36 ]; then 12 | sleep 900s 13 | fi 14 | fi 15 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/infra_tuning.sh: -------------------------------------------------------------------------------- 1 | mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.disabled 2 | mv /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel-testing.repo.disabled 3 | sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config 4 | setenforce 0 5 | 6 | cd /usr/lib/tuned/ 7 | cp -r throughput-performance/ gpfs-oci-performance 8 | 9 | echo " 10 | 11 | [main] 12 | summary=gpfs perf tuning for common gpfs workloads 13 | 14 | [cpu] 15 | force_latency=1 16 | governor=performance 17 | energy_perf_bias=performance 18 | min_perf_pct=100 19 | 20 | [vm] 21 | transparent_huge_pages=never 22 | 23 | [sysctl] 24 | net.ipv4.tcp_timestamps=1 25 | net.ipv4.tcp_sack=1 26 | net.ipv4.tcp_dsack=1 27 | net.ipv4.tcp_low_latency=1 28 | net.ipv4.tcp_adv_win_scale=2 29 | net.ipv4.tcp_window_scaling=1 30 | net.ipv4.tcp_slow_start_after_idle=0 31 | net.ipv4.tcp_syn_retries=8 32 | net.ipv4.tcp_rmem=4096 87380 16777216 33 | net.ipv4.tcp_wmem=4096 65536 16777216 34 | net.core.rmem_max=16777216 35 | net.core.wmem_max=16777216 36 | net.core.rmem_default=16777216 37 | net.core.wmem_default=16777216 38 | net.core.optmem_max=16777216 39 | net.core.somaxconn = 8192 40 | net.core.netdev_max_backlog=250000 41 | sunrpc.udp_slot_table_entries=128 42 | sunrpc.tcp_slot_table_entries=128 43 | kernel.sysrq = 1 44 | kernel.sched_min_granularity_ns = 10000000 45 | kernel.sched_wakeup_granularity_ns = 15000000 46 | vm.min_free_kbytes = 16777216 47 | vm.dirty_ratio = 30 48 | vm.dirty_background_ratio = 10 49 | vm.swappiness=30 50 | " > gpfs-oci-performance/tuned.conf 51 | 52 | cd - 53 | 54 | echo "$thisHost" | grep -q $nsdNodeHostnamePrefix 55 | if [ $? -eq 0 ] ; then 56 | tuned-adm profile gpfs-oci-performance 57 | fi 58 | 59 | 60 | # check for enough memory. 61 | echo "$thisHost" | grep -q $clientNodeHostnamePrefix 62 | if [ $? -eq 0 ] ; then 63 | coreIdCount=`grep "^core id" /proc/cpuinfo | sort -u | wc -l` ; echo $coreIdCount 64 | socketCount=`echo $(($(grep "^physical id" /proc/cpuinfo | awk '{print $4}' | sort -un | tail -1)+1))` ; echo $socketCount 65 | if [ $((socketCount*coreIdCount)) -gt 4 ]; then 66 | tuned-adm profile gpfs-oci-performance 67 | else 68 | # Client with less than 4 physical cores and less 30GB memory, above tuned profile requires atleast 16GB of vm.min_free_kbytes, hence let user do manual tuning. 69 | echo "skip profile tuning..." 70 | fi ; 71 | fi; 72 | 73 | tuned-adm active 74 | 75 | echo "$thisHost" | grep -q $clientNodeHostnamePrefix 76 | if [ $? -eq 0 ] ; then 77 | echo off > /sys/devices/system/cpu/smt/control 78 | fi 79 | 80 | echo "$thisHost" | egrep -q "$clientNodeHostnamePrefix|$nsdNodeHostnamePrefix" 81 | if [ $? -eq 0 ] ; then 82 | echo ' 83 | * soft memlock -1 84 | * hard memlock -1 85 | * soft rss -1 86 | * hard rss -1 87 | * soft core -1 88 | * hard core -1 89 | * soft maxlogins 8192 90 | * hard maxlogins 8192 91 | * soft stack -1 92 | * hard stack -1 93 | * soft nproc 2067554 94 | * hard nproc 2067554 95 | * soft nofile 500000 96 | * hard nofile 500000 97 | ' >> /etc/security/limits.conf 98 | 99 | echo ' 100 | ulimit -l unlimited 101 | ulimit -m unlimited 102 | ulimit -c unlimited 103 | ulimit -s unlimited 104 | ulimit -u 2067554 105 | ulimit -n 500000 106 | ' >> ~/.bash_profile 107 | 108 | fi 109 | 110 | 111 | echo "$thisHost" | grep -q $nsdNodeHostnamePrefix 112 | if [ $? -eq 0 ] ; then 113 | 114 | echo '# 115 | # Identify eligible SCSI disks by the absence of a SWAP partition. 116 | 117 | ACTION=="add|change", SUBSYSTEM=="block", KERNEL=="sd*[^0-9]", PROGRAM="/usr/bin/lsblk -rno FSTYPE,MOUNTPOINT,NAME /dev/%k", RESULT!="*SWAP*", ATTR{queue/scheduler}="deadline", ATTR{queue/nr_requests}="256", ATTR{device/queue_depth}="31", ATTR{queue/max_sectors_kb}="8192", ATTR{queue/read_ahead_kb}="0", ATTR{queue/rq_affinity}="2" 118 | ' > /etc/udev/rules.d/99-ibm-spectrum-scale.rules 119 | # reload the rules 120 | udevadm control --reload-rules && udevadm trigger 121 | 122 | fi 123 | 124 | cd - 125 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/install_spectrum_scale.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | cd /tmp/ 4 | curl -O $downloadUrl -s 5 | 6 | while [ $? -ne 0 ]; do 7 | rm -rf /tmp/Spectrum_Scale_Data_Management-* 8 | rm -rf "/tmp/Spectrum Scale*" 9 | curl -O $downloadUrl -s 10 | done 11 | 12 | echo $downloadUrl | grep "Developer" | grep "zip$" 13 | if [ $? -eq 0 ]; then 14 | SS_DE=true 15 | zip_filepath=`ls /tmp/* | grep "Developer" | grep "${version}" | grep "zip$" ` 16 | unzip "$zip_filepath" 17 | install_dir=`ls -d /tmp/*/ | grep "Developer" | grep "Edition" ` 18 | cd """$install_dir""" 19 | cp Spectrum_Scale_Developer-${version}-x86_64-Linux-install /tmp/ 20 | install_filepath="/tmp/Spectrum_Scale_Developer-${version}-x86_64-Linux-install" 21 | else 22 | SS_DE=false 23 | install_filepath="/tmp/Spectrum_Scale_Data_Management-${version}-x86_64-Linux-install" 24 | fi 25 | 26 | while [ ! -f $install_filepath ]; 27 | do 28 | sleep 5s 29 | echo "Waiting for download" 30 | done 31 | 32 | chmod +x $install_filepath 33 | $install_filepath --silent 34 | 35 | 36 | echo "$version" > /etc/yum/vars/spec_scale_ver 37 | 38 | echo '[spectrum_scale-gpfs] 39 | name = Spectrum Scale - GPFS 40 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/gpfs_rpms 41 | gpgcheck=0 42 | enabled=1 43 | [spectrum_scale-ganesha] 44 | name = Spectrum Scale - NFS-Ganesha 45 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/ganesha_rpms/rhel7 46 | gpgcheck=0 47 | enabled=1 48 | [spectrum_scale-smb] 49 | name = Spectrum Scale - SMB 50 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/smb_rpms/rhel7 51 | gpgcheck=0 52 | enabled=1 53 | [spectrum_scale-zimon] 54 | name = Spectrum Scale - Zimon 55 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/zimon_rpms/rhel7 56 | gpgcheck=0 57 | enabled=1 58 | [spectrum_scale-gpfs-optional] 59 | name = Spectrum Scale - GPFS optional 60 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/gpfs_rpms/rhel 61 | gpgcheck=0 62 | enabled=1 63 | ' > /etc/yum.repos.d/spectrum-scale.repo 64 | 65 | echo '[spectrum_scale-object-rhel8] 66 | name = Spectrum Scale - Object 67 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/object_rpms/rhel8 68 | gpgcheck=0 69 | enabled=1 70 | [spectrum_scale-gpfs-rhel] 71 | name = Spectrum Scale - rhel 72 | baseurl = file:///usr/lpp/mmfs/$spec_scale_ver/gpfs_rpms/rhel 73 | gpgcheck=0 74 | enabled=1 75 | ' >> /etc/yum.repos.d/spectrum-scale.repo 76 | 77 | 78 | 79 | yum clean all 80 | yum makecache 81 | rerun=false 82 | yum -y install cpp gcc gcc-c++ binutils 83 | if [ $? -ne 0 ]; then 84 | rerun=true 85 | fi 86 | 87 | function downloadKernelRPMs { 88 | packagePrefix=$1 89 | kernelVersion=`uname -a | gawk -F" " '{ print $3 }' ` ; echo $kernelVersion 90 | sudo yum install -y -q redhat-lsb-core 91 | lsb_release -a 92 | osVersion=`lsb_release -a | grep "Release:" | gawk -F" " '{ print $2 }' | gawk -F"." '{ print $1"."$2 }' ` ; echo $osVersion 93 | fullOSReleaseVersion=`lsb_release -a | grep "Release:" | gawk -F" " '{ print $2 }'` ; echo $fullOSReleaseVersion 94 | 95 | declare -a rpmServers=("http://linuxsoft.cern.ch/centos-vault/${fullOSReleaseVersion}/updates/x86_64/Packages" 96 | "http://repo1.xorcom.com/repos/centos/7/x86_64/Updates_OS_X86_64/Packages/k" 97 | "http://ftp.scientificlinux.org/linux/scientific/${osVersion}/x86_64/updates/security" 98 | "http://archive.kernel.org/centos-vault/${fullOSReleaseVersion}/updates/x86_64/Packages" 99 | ) 100 | 101 | for rpmDownloadURLPrefix in "${rpmServers[@]}" 102 | do 103 | echo "$rpmDownloadURLPrefix" 104 | curl --head --fail --silent ${rpmDownloadURLPrefix}/${packagePrefix}-${kernelVersion}.rpm 105 | if [ $? -eq 0 ]; then 106 | curl -O ${rpmDownloadURLPrefix}/${packagePrefix}-${kernelVersion}.rpm 107 | if [ $? -eq 0 ]; then 108 | break; 109 | fi 110 | fi 111 | done 112 | } 113 | kernelVersion=`uname -a | gawk -F" " '{ print $3 }' ` ; echo $kernelVersion 114 | downloadKernelRPMs "kernel-devel" 115 | downloadKernelRPMs "kernel-headers" 116 | # --oldpackage 117 | rpm -Uvh kernel-devel-${kernelVersion}.rpm --oldpackage 118 | rpm -Uvh kernel-headers-${kernelVersion}.rpm --oldpackage 119 | 120 | if [ "$rerun" = "true" ]; then 121 | yum -y install cpp gcc gcc-c++ binutils 122 | fi 123 | 124 | yum -y install psmisc numad numactl iperf3 dstat iproute automake autoconf git 125 | 126 | echo "$thisHost" | grep -q $cesNodeHostnamePrefix 127 | if [ $? -eq 0 ] ; then 128 | # AD 129 | yum install -y nfs-utils bind-utils 130 | # LDAP 131 | yum install -y nfs-utils openldap-client sssd-common sssd-ldap 132 | fi 133 | 134 | echo "$thisHost" | grep -q $mgmtGuiNodeHostnamePrefix 135 | if [ $? -eq 0 ] ; then 136 | #GUI node: 137 | yum -y install gpfs.base gpfs.gpl gpfs.msg.en_US gpfs.gskit gpfs.license* gpfs.ext gpfs.crypto gpfs.compression gpfs.adv gpfs.gss.pmsensors gpfs.docs gpfs.java gpfs.kafka gpfs.librdkafka gpfs.gui gpfs.gss.pmcollector 138 | else 139 | yum -y install gpfs.base gpfs.gpl gpfs.msg.en_US gpfs.gskit gpfs.license* gpfs.ext gpfs.crypto gpfs.compression gpfs.adv gpfs.gss.pmsensors gpfs.docs gpfs.java gpfs.kafka gpfs.librdkafka 140 | fi 141 | 142 | sed -i '/distroverpkg/a exclude=kernel*' /etc/yum.conf 143 | 144 | 145 | echo "cloud-init complete" 146 | touch /tmp/cloud_init.complete 147 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/nodes-cloud-init-complete-status-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | hostname 4 | while [ ! -f /tmp/cloud_init.complete ] 5 | do 6 | sleep 60s 7 | echo "Waiting for compute node: `hostname --fqdn` cloud-init to complete ..." 8 | done 9 | 10 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/nsddevices: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | # @(#)53 1.5 src/avs/fs/mmfs/ts/config/nsddevices.sample, mmfs, avs_rfks2, rfks21523a 8/9/04 16:51:23 3 | ############################################################################## 4 | # 5 | # When properly installed, this script is invoked by the 6 | # /usr/lpp/mmfs/bin/mmdevdiscover script. 7 | # 8 | # INSTALLATION GUIDELINES FOR THIS SCRIPT: 9 | # 10 | # a) edit this script using the configuration guidelines below 11 | # b) copy this script to /var/mmfs/etc/nsddevices 12 | # c) ensure this script is executable (chmod +x /var/mmfs/etc/nsddevices) 13 | # 14 | # DESCRIPTION OF NSD DEVICE DISCOVERY: 15 | # 16 | # The mmdevdiscover script and conversely this script are invoked by 17 | # the mmfsd daemon when it tries to discover or verify physical 18 | # devices previously defined to GPFS with the mmcrnsd command. These 19 | # scripts identify devices found in the /dev file system on the local 20 | # machine that may correlate to NSDs defined to GPFS. 21 | # 22 | # GPFS uses the list of devices output by these scripts in mapping 23 | # the NSD name listed in the configuration database to a local device 24 | # in the /dev file system. When an NSD is created via the mmcrnsd 25 | # command it is marked with a unique identifier written to sector 26 | # two of the device. This unique identifier is recorded in the 27 | # configuration database along with the user recognizable NSD name. 28 | # 29 | # During GPFS disk discovery each device name output by mmdevdiscover 30 | # and nsddevices is opened in turn and sector two of each device is 31 | # read. If a match between an NSD identifier on the device and an 32 | # identifier recorded in the configuration database is found, then 33 | # this machine has local access to the NSD device. I/O is thus 34 | # subsequently performed via this local /dev interface. 35 | # 36 | # CONFIGURATION AND EDITING GUIDELINES: 37 | # 38 | # If this script is not installed then disk discovery is done 39 | # only via the commands listed in mmdevdiscover. 40 | # 41 | # If this script is installed and returns a NON ZERO return code 42 | # then the disk discovery commands listed in mmdevdiscover will ALSO 43 | # be run. 44 | # 45 | # If this script is installed and returns a ZERO return code 46 | # then the disk discovery commands listed in mmdevdiscover will NOT 47 | # be run. 48 | # 49 | # The output from both this script and nsddevices is a number 50 | # of lines in the following format: 51 | # 52 | # deviceName deviceType 53 | # 54 | # where (deviceName) is a device name such as (hdisk1) 55 | # and (deviceType) is a set of known disk types. Consult 56 | # 57 | # /usr/lpp/mmfs/bin/mmdevdiscover 58 | # 59 | # for a list of currently known deviceTypes 60 | # 61 | # Example output: 62 | # 63 | # hdisk1 hdisk 64 | # hdisk2 hdisk 65 | # 66 | ############################################################################## 67 | 68 | osName=$(/bin/uname -s) 69 | 70 | if [[ $osName = Linux ]] 71 | then 72 | #ls /dev/mapper|grep map|awk '{print "mapper/"$1,"dmm"}' 73 | ls /dev/oracleoci | grep oraclevd| awk '{print "oracleoci/"$1,"dmm"}' 74 | fi 75 | 76 | if [[ $osName = AIX ]] 77 | then 78 | : # Add function to discover disks in the AIX environment. 79 | fi 80 | 81 | # To bypass the GPFS disk discovery (/usr/lpp/mmfs/bin/mmdevdiscover), 82 | return 0 83 | 84 | # To continue with the GPFS disk discovery steps, 85 | # return 1 86 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/passwordless_ssh.sh: -------------------------------------------------------------------------------- 1 | ##echo "$sshPrivateKey" > /root/.ssh/id_rsa 2 | ##echo "$sshPublicKey" > /root/.ssh/id_rsa.pub 3 | 4 | while [ ! -f /home/opc/.ssh/id_rsa ] 5 | do 6 | sleep 60s 7 | echo "Waiting for file to be copied by TF ..." 8 | done 9 | 10 | \cp /home/opc/.ssh/id_rsa* /root/.ssh/ 11 | if [ $? -ne 0 ]; then 12 | exit 1; 13 | fi 14 | 15 | 16 | chmod 600 /home/opc/.ssh/id_rsa* 17 | chmod 640 /home/opc/.ssh/authorized_keys 18 | 19 | cp /etc/ssh/sshd_config /etc/ssh/sshd_config.backup 20 | sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config 21 | 22 | mv /root/.ssh/authorized_keys /root/.ssh/authorized_keys.backup 23 | cp /home/opc/.ssh/authorized_keys /root/.ssh/authorized_keys 24 | ##cd /root/.ssh/; cat id_rsa.pub >> authorized_keys ; cd - 25 | cat /root/.ssh/id_rsa.pub >> authorized_keys ; 26 | 27 | sudo chmod 600 /root/.ssh/id_rsa* 28 | sudo chmod 640 /root/.ssh/authorized_keys 29 | 30 | 31 | find_cluster_nodes () { 32 | echo "Doing nslookup for $nodeType nodes" 33 | ct=1 34 | if [ $nodeCount -gt 0 ]; then 35 | while [ $ct -le $nodeCount ]; do 36 | nslk=`nslookup $nodeHostnamePrefix${ct}.$domainName` 37 | ns_ck=`echo -e $?` 38 | if [ $ns_ck = 0 ]; then 39 | hname=`nslookup $nodeHostnamePrefix${ct}.$domainName | grep Name | gawk '{print $2}'` 40 | echo "$hname" >> /tmp/${nodeType}nodehosts; 41 | echo "$hname" >> /tmp/allnodehosts; 42 | ct=$((ct+1)); 43 | else 44 | echo "Sleeping for 10 secs and will check again for nslookup $nodeHostnamePrefix${ct}.$domainName" 45 | sleep 10 46 | fi 47 | done; 48 | echo "Found `cat /tmp/${nodeType}nodehosts | wc -l` $nodeType nodes"; 49 | echo `cat /tmp/${nodeType}nodehosts`; 50 | else 51 | echo "no $nodeType nodes configured" 52 | fi 53 | } 54 | 55 | # Subnet used for GPFS network 56 | domainName=${privateBSubnetsFQDN} 57 | nodeType="nsd" 58 | nodeHostnamePrefix=$nsdNodeHostnamePrefix 59 | nodeCount=$nsdNodeCount 60 | find_cluster_nodes 61 | 62 | nodeType="client" 63 | nodeHostnamePrefix=$clientNodeHostnamePrefix 64 | nodeCount=$clientNodeCount 65 | find_cluster_nodes 66 | 67 | nodeType="ces" 68 | nodeHostnamePrefix=$cesNodeHostnamePrefix 69 | nodeCount=$cesNodeCount 70 | find_cluster_nodes 71 | 72 | nodeType="mgmtgui" 73 | nodeHostnamePrefix=$mgmtGuiNodeHostnamePrefix 74 | nodeCount=$mgmtGuiNodeCount 75 | find_cluster_nodes 76 | 77 | if [ ! -f ~/.ssh/known_hosts ]; then 78 | touch ~/.ssh/known_hosts 79 | fi 80 | 81 | do_ssh_keyscan () { 82 | if [ -z `ssh-keygen -F $host` ]; then 83 | ssh-keyscan -H $host > /tmp/keyscan 84 | cat /tmp/keyscan | grep "ssh-rsa" 85 | while [ $? -ne 0 ]; do 86 | sleep 10s; 87 | ssh-keyscan -H $host > /tmp/keyscan 88 | cat /tmp/keyscan | grep "ssh-rsa" 89 | done; 90 | ssh-keyscan -H $host >> ~/.ssh/known_hosts 91 | fi 92 | } 93 | 94 | # passwordless ssh 95 | for host_fqdn in `cat /tmp/allnodehosts` ; do 96 | host=$host_fqdn 97 | do_ssh_keyscan 98 | host=${host_fqdn%%.*} 99 | do_ssh_keyscan 100 | host_ip=`nslookup $host_fqdn | grep "Address: " | gawk '{print $2}'` 101 | host=$host_ip 102 | do_ssh_keyscan 103 | echo "$host_ip ${host_fqdn} ${host_fqdn%%.*}" >> /etc/hosts 104 | done ; 105 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/set_env_variables.sh: -------------------------------------------------------------------------------- 1 | 2 | echo " 3 | version=\"$version\" 4 | downloadUrl=\"$downloadUrl\" 5 | sshPrivateKey=\"$sshPrivateKey\" 6 | sshPublicKey=\"$sshPublicKey\" 7 | totalNsdNodePools=\"$totalNsdNodePools\" 8 | nsdNodesPerPool=\"$nsdNodesPerPool\" 9 | nsdNodeCount=\"$nsdNodeCount\" 10 | nsdNodeHostnamePrefix=\"$nsdNodeHostnamePrefix\" 11 | clientNodeCount=\"$clientNodeCount\" 12 | clientNodeHostnamePrefix=\"$clientNodeHostnamePrefix\" 13 | blockSize=\"$blockSize\" 14 | dataReplica=\"$dataReplica\" 15 | metadataReplica=\"$metadataReplica\" 16 | gpfsMountPoint=\"$gpfsMountPoint\" 17 | highAvailability=\"$highAvailability\" 18 | sharedDataDiskCount=\"$sharedDataDiskCount\" 19 | blockVolumesPerPool=\"$blockVolumesPerPool\" 20 | installerNode=\"$installerNode\" 21 | vcnFQDN=\"$vcnFQDN\" 22 | privateSubnetsFQDN=\"$privateSubnetsFQDN\" 23 | privateBSubnetsFQDN=\"$privateBSubnetsFQDN\" 24 | companyName=\"$companyName\" 25 | companyID=\"$companyID\" 26 | countryCode=\"$countryCode\" 27 | emailaddress=\"$emailaddress\" 28 | cesNodeCount=\"$cesNodeCount\" 29 | cesNodeHostnamePrefix=\"$cesNodeHostnamePrefix\" 30 | mgmtGuiNodeCount=\"$mgmtGuiNodeCount\" 31 | mgmtGuiNodeHostnamePrefix=\"$mgmtGuiNodeHostnamePrefix\" 32 | privateProtocolSubnetFQDN=\"$privateProtocolSubnetFQDN\" 33 | " > /tmp/gpfs_env_variables.sh 34 | 35 | # we might need this for BM shapes for core OS services to be ready 36 | sleep 60s 37 | 38 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/scripts/update_resolv_conf.sh: -------------------------------------------------------------------------------- 1 | mv /etc/resolv.conf /etc/resolv.conf.backup 2 | echo "search ${vcnFQDN} ${privateBSubnetsFQDN} ${privateSubnetsFQDN} ${privateProtocolSubnetFQDN}" > /etc/resolv.conf 3 | echo "nameserver 169.254.169.254" >> /etc/resolv.conf 4 | 5 | if [ -z /etc/oci-hostname.conf ]; then 6 | echo "PRESERVE_HOSTINFO=2" > /etc/oci-hostname.conf 7 | else 8 | # https://docs.cloud.oracle.com/iaas/Content/Network/Tasks/managingDHCP.htm#notes 9 | sed -i "s/^PRESERVE_HOSTINFO/#PRESERVE_HOSTINFO/g" /etc/oci-hostname.conf 10 | echo "PRESERVE_HOSTINFO=2" >> /etc/oci-hostname.conf 11 | fi 12 | # not be overwritten by dhclient 13 | chattr +i /etc/resolv.conf 14 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/secondary_vnic.tf: -------------------------------------------------------------------------------- 1 | resource "oci_core_vnic_attachment" "secondary_vnic_attachment" { 2 | #1# count = (local.dual_nics ? var.total_nsd_node_pools * var.nsd_nodes_per_pool : var.total_nsd_node_pools * var.nsd_nodes_per_pool) 3 | count = (local.dual_vnic ? var.total_nsd_node_pools * var.nsd_nodes_per_pool : 0) 4 | create_vnic_details { 5 | #1# subnet_id = oci_core_subnet.privateb.*.id[0] 6 | subnet_id = local.fs_subnet_id 7 | 8 | assign_public_ip = "false" 9 | display_name = "${var.nsd_node_hostname_prefix}${format("%01d", count.index+1)}" 10 | hostname_label = "${var.nsd_node_hostname_prefix}${format("%01d", count.index+1)}" 11 | # false is default value 12 | skip_source_dest_check = "false" 13 | } 14 | instance_id = element(oci_core_instance.nsd_node.*.id, count.index) 15 | 16 | #Optional 17 | display_name = "SecondaryVNIC" 18 | # set to 1, if you want to use 2nd physical NIC for this VNIC 19 | ####nic_index = "1" 20 | #1# nic_index = (local.dual_nics ? (local.dual_nics_hpc_shape ? "0" : "1") : "0") 21 | nic_index = (local.dual_vnic ? "1" : "0") 22 | 23 | } 24 | 25 | # dual_nics_ces_node 26 | resource "oci_core_vnic_attachment" "ces_node_secondary_vnic_attachment" { 27 | # Assume if CES node is needed, a seperate protocol subnet is required 28 | count = (local.dual_nics_ces_node ? var.ces_node_count : var.ces_node_count) 29 | #! Change logic to use the below, once whether to run CES node on protocol subnet or not is made 30 | #! count = (local.dual_vnic_ces ? var.ces_node["node_count"] : 0) 31 | create_vnic_details { 32 | #1# subnet_id = oci_core_subnet.privateb.*.id[0] 33 | subnet_id = local.fs_subnet_id 34 | 35 | assign_public_ip = "false" 36 | display_name = "${var.ces_node_hostname_prefix}${format("%01d", count.index+1)}" 37 | hostname_label = "${var.ces_node_hostname_prefix}${format("%01d", count.index+1)}" 38 | # false is default value 39 | skip_source_dest_check = "false" 40 | } 41 | instance_id = element(oci_core_instance.ces_node.*.id, count.index) 42 | 43 | #Optional 44 | display_name = "SecondaryVNIC" 45 | # set to 1, if you want to use 2nd physical NIC for this VNIC 46 | #### nic_index = "1" 47 | nic_index = (local.dual_nics_ces_node ? (local.dual_nics_ces_hpc_shape ? "0" : "1") : "0") 48 | } 49 | 50 | 51 | # virtual IP addresses for the CES IP address pool 52 | resource "oci_core_vnic_attachment" "ces_node_virtual_ip_pool_secondary_vnic_attachment" { 53 | depends_on = [ oci_core_vnic_attachment.ces_node_secondary_vnic_attachment] 54 | count = var.ces_node_count 55 | create_vnic_details { 56 | #1# subnet_id = oci_core_subnet.privateprotocol.*.id[0] 57 | subnet_id = local.protocol_subnet_id 58 | 59 | assign_public_ip = "false" 60 | display_name = "${var.ces_node_hostname_prefix}vip-pool-${format("%01d", count.index+1)}" 61 | hostname_label = "${var.ces_node_hostname_prefix}vip-pool-${format("%01d", count.index+1)}" 62 | # false is default value 63 | skip_source_dest_check = "false" 64 | } 65 | instance_id = element(oci_core_instance.ces_node.*.id, count.index) 66 | 67 | #Optional 68 | display_name = "SecondaryVNIC" 69 | # set to 1, if you want to use 2nd physical NIC for this VNIC 70 | nic_index = "0" 71 | } 72 | 73 | 74 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/shared_data_block_volume.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | shared_data_disk_count = (var.total_nsd_node_pools * var.block_volumes_per_pool) 3 | } 4 | 5 | resource "oci_core_volume" "shared_data_block_volume" { 6 | count = var.total_nsd_node_pools * var.block_volumes_per_pool 7 | availability_domain = local.ad 8 | #lookup(data.oci_identity_availability_domains.ADs.availability_domains[((count.index < (local.shared_data_disk_count / 2)) ? local.site1 : local.site2)],"name") 9 | compartment_id = var.compartment_ocid 10 | display_name = "SharedData${count.index+1}" 11 | size_in_gbs = var.nsd_size 12 | vpus_per_gb = var.volume_type_vpus_per_gb_mapping[var.nsd_perf_tier] 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/userdata/cloudinit.ps1: -------------------------------------------------------------------------------- 1 | #ps1_sysnative 2 | 3 | # Template variables 4 | $user='${instance_user}' 5 | $password='${instance_password}' 6 | $computerName='${instance_name}' 7 | 8 | Write-Output "Changing $user password" 9 | net user $user $password 10 | Write-Output "Changed $user password" 11 | 12 | Write-Output "Configuring WinRM" 13 | # Allow unencrypted if you wish to use http 5985 endpoint 14 | winrm set winrm/config/service '@{AllowUnencrypted="true"}' 15 | 16 | # Create a self-signed certificate to configure WinRM for HTTPS 17 | $cert = New-SelfSignedCertificate -CertStoreLocation 'Cert:\LocalMachine\My' -DnsName $computerName 18 | Write-Output "Self-signed SSL certificate generated with details: $cert" 19 | 20 | $valueSet = @{ 21 | Hostname = $computerName 22 | CertificateThumbprint = $cert.Thumbprint 23 | } 24 | 25 | $selectorSet = @{ 26 | Transport = "HTTPS" 27 | Address = "*" 28 | } 29 | 30 | # Remove any prior HTTPS listener 31 | $listeners = Get-ChildItem WSMan:\localhost\Listener 32 | If (!($listeners | Where {$_.Keys -like "TRANSPORT=HTTPS"})) 33 | { 34 | Remove-WSManInstance -ResourceURI 'winrm/config/Listener' -SelectorSet $selectorSet 35 | } 36 | 37 | Write-Output "Enabling HTTPS listener" 38 | New-WSManInstance -ResourceURI 'winrm/config/Listener' -SelectorSet $selectorSet -ValueSet $valueSet 39 | Write-Output "Enabled HTTPS listener" 40 | 41 | Write-Output "Configured WinRM" 42 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/userdata/cloudinit.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | write_files: 3 | - path: C:/init.ps1 4 | content: | 5 | echo "add your custom commands and scripts here" 6 | # Depending on the image that is being used runcmd plugin may not be supported, you may use ps1_sysnative config instead 7 | # Windows-Server-2012-R2-Standard-Edition-VM-2018.07.19-0 - cloudbaseinit.plugins.common.userdataplugins.cloudconfig [-] Plugin 'runcmd' is currently not supported 8 | runcmd: 9 | - echo "Hello from Terraform" 10 | - powershell.exe "C:/init.ps1" 11 | 12 | # Depending on the image that is being used script plugin may not be supported, you may use ps1_sysnative config instead 13 | # Windows-Server-2012-R2-Standard-Edition-VM-2018.07.19-0 - cloudbaseinit.plugins.common.userdataplugins.cloudconfig [-] Plugin 'script' is currently not supported 14 | script: | 15 | 16 | winrm set winrm/config/client/auth '@{Basic="true"}' 17 | winrm set winrm/config/service/auth '@{Basic="true"}' 18 | winrm set winrm/config/service '@{AllowUnencrypted="true"}' 19 | winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="300"}' 20 | winrm set winrm/config '@{MaxTimeoutms="1800000"}' 21 | $file = Join-Path -Path $env:SystemRoot -ChildPath (Get-Date).ToString("MM-dd-yy-hh-mm") 22 | New-Item $file -ItemType file -Value "Hello from Terraform" 23 | 24 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/userdata/setup.ps1: -------------------------------------------------------------------------------- 1 | $ipv4='${volume_ipv4}' 2 | $iqn='${volume_iqn}' 3 | 4 | echo 'Hello from terraform' > C:\hello_from_terraform_1234567890.txt 5 | 6 | Write-Output 'Configuring ISCSI for Block Volumes' 7 | 8 | Set-Service -Name msiscsi -StartupType Automatic 9 | Start-Service msiscsi 10 | 11 | New-IscsiTargetPortal -TargetPortalAddress $ipv4 12 | Connect-IscsiTarget -NodeAddress $iqn -TargetPortalAddress $ipv4 -IsPersistent $True 13 | 14 | Write-Output 'Configured ISCSI for Block Volumes' 15 | 16 | Write-Output 'Configuring the new disk for a partition and file system' 17 | Get-Disk -Number 1 | Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem NTFS -NewFileSystemLabel "tfdisk" -Confirm:$false 18 | Write-Output 'Configured the new disk' 19 | -------------------------------------------------------------------------------- /network_shared_disk_server_model/windows_smb_client.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | // Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. 5 | 6 | ############ 7 | # Cloudinit 8 | ############ 9 | # Generate a new strong password for your instance 10 | resource "random_string" "instance_password" { 11 | length = 16 12 | special = true 13 | } 14 | 15 | # Use the cloudinit.ps1 as a template and pass the instance name, user and password as variables to same 16 | data "template_file" "cloudinit_ps1" { 17 | vars = { 18 | instance_user = "opc" 19 | instance_password = random_string.instance_password.result 20 | instance_name = "${var.windows_smb_client_hostname_prefix}1" 21 | } 22 | 23 | template = file("${var.userdata}/${var.cloudinit_ps1}") 24 | } 25 | 26 | data "template_cloudinit_config" "cloudinit_config" { 27 | gzip = false 28 | base64_encode = true 29 | 30 | # The cloudinit.ps1 uses the #ps1_sysnative to update the instance password and configure winrm for https traffic 31 | part { 32 | filename = var.cloudinit_ps1 33 | content_type = "text/x-shellscript" 34 | content = data.template_file.cloudinit_ps1.rendered 35 | } 36 | 37 | # The cloudinit.yml uses the #cloud-config to write files remotely into the instance, this is executed as part of instance setup 38 | part { 39 | filename = var.cloudinit_config 40 | content_type = "text/cloud-config" 41 | content = file("${var.userdata}/${var.cloudinit_config}") 42 | } 43 | } 44 | 45 | ########### 46 | # Compute 47 | ########### 48 | resource "oci_core_instance" "windows_smb_client" { 49 | count = var.windows_smb_client_node_count 50 | availability_domain = local.ad 51 | 52 | fault_domain = "FAULT-DOMAIN-${(count.index%3)+1}" 53 | compartment_id = var.compartment_ocid 54 | display_name = "${var.windows_smb_client_hostname_prefix}${format("%01d", count.index + 1)}" 55 | shape = var.windows_smb_client_shape 56 | 57 | create_vnic_details { 58 | subnet_id = oci_core_subnet.public.*.id[0] 59 | skip_source_dest_check = true 60 | hostname_label = "${var.windows_smb_client_hostname_prefix}${format("%01d", count.index + 1)}" 61 | } 62 | 63 | 64 | # Refer cloud-init in https://docs.cloud.oracle.com/iaas/api/#/en/iaas/20160918/datatypes/LaunchInstanceDetails 65 | metadata = { 66 | # Base64 encoded YAML based user_data to be passed to cloud-init 67 | user_data = data.template_cloudinit_config.cloudinit_config.rendered 68 | } 69 | 70 | source_details { 71 | boot_volume_size_in_gbs = var.windows_smb_client_boot_volume_size_in_gbs 72 | source_id = var.w_images[var.region] 73 | source_type = "image" 74 | } 75 | 76 | } 77 | 78 | 79 | data "oci_core_instance_credentials" "InstanceCredentials" { 80 | count = var.windows_smb_client_node_count 81 | # depends_on was added as a workaround to TF issue with empty oci_core_instance.windows_smb_client.*.id[0] 82 | depends_on = [ oci_core_instance.windows_smb_client ] 83 | instance_id = (var.windows_smb_client_node_count > 0 ? element(concat(oci_core_instance.windows_smb_client.*.id, [""]), 0) : "") 84 | } 85 | 86 | 87 | ########## 88 | # Outputs 89 | ########## 90 | 91 | output "Windows-SMB-Client-Node-Details" { 92 | value = < 0 ? oci_core_instance.windows_smb_client.*.public_ip[0] : ""} 97 | InstancePrivateIP: ${var.windows_smb_client_node_count > 0 ? oci_core_instance.windows_smb_client.*.private_ip[0] : ""} 98 | END 99 | } 100 | 101 | -------------------------------------------------------------------------------- /network_shared_disk_server_model_latest.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle-quickstart/oci-ibm-spectrum-scale/e38a96bdd8ee9114df39695102eb3f5452f878c2/network_shared_disk_server_model_latest.zip --------------------------------------------------------------------------------