├── .github └── ISSUE_TEMPLATE ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── bashsource.tf ├── datasources.tf ├── docs ├── cluster-access.md ├── example-deployments.md ├── examples.md ├── gpu-workers.md ├── images │ ├── arch.jpg │ ├── private_cp_subnet_private_lb_access.jpg │ ├── private_cp_subnet_public_lb_access.jpg │ ├── private_cp_subnet_public_lb_failure.jpg │ └── public_cp_subnet_access.jpg └── input-variables.md ├── identity ├── cloud_controller_user.tf ├── flexvolume_user.tf ├── outputs.tf ├── provider.tf ├── variables.tf └── volume_provisioner_user.tf ├── instances ├── etcd │ ├── cloud_init │ │ └── bootstrap.template.sh │ ├── datasources.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── k8smaster │ ├── cloud_init │ │ └── bootstrap.template.yaml │ ├── datasources.tf │ ├── main.tf │ ├── manifests │ │ ├── kube-apiserver.yaml │ │ ├── kube-controller-manager.yaml │ │ ├── kube-dns.yaml │ │ ├── kube-proxy.yaml │ │ ├── kube-rbac-role-binding.yaml │ │ ├── kube-scheduler.yaml │ │ ├── kubernetes-dashboard.yaml │ │ └── master-kubeconfig.template.yaml │ ├── outputs.tf │ ├── scripts │ │ ├── kubelet.service │ │ ├── setup.preflight.sh │ │ ├── setup.template.sh │ │ └── token_auth.csv │ └── variables.tf └── k8sworker │ ├── cloud_init │ └── bootstrap.template.yaml │ ├── datasources.tf │ ├── main.tf │ ├── manifests │ ├── kube-proxy.template.yaml │ └── worker-kubeconfig.template.yaml │ ├── output.tf │ ├── scripts │ ├── kubelet.service │ ├── setup.preflight.sh │ └── setup.template.sh │ └── variables.tf ├── k8s-oci.tf ├── kubernetes ├── kubeconfig │ ├── kubeconfig.tf │ ├── output.tf │ └── variables.tf ├── oci-cloud-controller │ ├── cloud-provider-secret.yaml │ ├── cloud-provider.json │ ├── datasources.tf │ ├── output.tf │ └── variables.tf ├── oci-flexvolume-driver │ ├── config.yaml │ ├── datasources.tf │ ├── output.tf │ └── variables.tf └── oci-volume-provisioner │ ├── config.yaml │ ├── datasources.tf │ ├── output.tf │ └── variables.tf ├── network ├── loadbalancers │ ├── etcd │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf │ ├── k8smaster │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf │ └── reverse-proxy │ │ ├── datasources.tf │ │ ├── output.tf │ │ ├── templates │ │ ├── clount_init.yaml │ │ ├── nginx.conf │ │ └── setup.sh │ │ └── variable.tf └── vcn │ ├── cloud_init │ └── bootstrap.template.yaml │ ├── datasources.tf │ ├── natinstance.tf │ ├── outputs.tf │ ├── securitylists.tf │ ├── subnets.tf │ ├── variables.tf │ └── vcn.tf ├── outputs.tf ├── provider.tf ├── scripts └── cluster-check.sh ├── terraform.example.tfvars ├── tests ├── README.md ├── create │ └── runner.py ├── requirements.txt └── resources │ ├── configs │ └── public-cluster.tfvars │ ├── frontend-service.yml │ └── hello-service.yml ├── tls ├── main.tf ├── outputs.tf └── variables.tf ├── variables.tf └── wercker.yml /.github/ISSUE_TEMPLATE: -------------------------------------------------------------------------------- 1 | ### Terraform Version 2 | 3 |
4 | # Run this command to get the terraform version: 5 | 6 | $ terraform -v 7 | 8 |
9 | 10 | ### OCI Provider Version 11 | 12 |
13 | # Execute the plugin directly to get the version: 14 | 15 | $ \/terraform-provider-oci 16 | 17 |
18 | 19 | ### Terraform Installer for Kubernetes Version 20 | 21 |
22 | # The version/tag/release or commit hash (of this project) the issue occurred on 23 |
24 | 25 | ### Input Variables 26 | 27 |
28 | # Values of non-sensitive input variables 29 | 30 |
31 | 32 | ### Description of issue: 33 | 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .tfstate 2 | 3 | # intellij 4 | *.iml 5 | *.ipr 6 | *.iws 7 | .idea 8 | 9 | **/.terraform 10 | **/.wercker 11 | **/terraform.tfstate 12 | **/terraform.tfstate.backup 13 | terraform.tfvars 14 | **/*~ 15 | generated/** 16 | source.sh 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to the Terraform Kubernetes Installer for Oracle Bare Metal Cloud Services 2 | 3 | Pull requests can be made under 4 | [The Oracle Contributor Agreement](https://www.oracle.com/technetwork/community/oca-486395.html) 5 | (OCA). 6 | 7 | For pull requests to be accepted, the bottom of 8 | your commit message must have the following line using your name and 9 | e-mail address as it appears in the OCA Signatories list. 10 | 11 | ``` 12 | Signed-off-by: Your Name 13 | ``` 14 | 15 | This can be automatically added to pull requests by committing with: 16 | 17 | ``` 18 | git commit --signoff 19 | ```` 20 | 21 | Only pull requests from committers that can be verified as having 22 | signed the OCA can be accepted. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [terraform]: https://terraform.io 2 | [oci]: https://cloud.oracle.com/cloud-infrastructure 3 | [oci provider]: https://github.com/oracle/terraform-provider-oci/releases 4 | [API signing]: https://docs.us-phoenix-1.oraclecloud.com/Content/API/Concepts/apisigningkey.htm 5 | [Kubectl]: https://kubernetes.io/docs/tasks/tools/install-kubectl/ 6 | 7 | # Terraform Kubernetes Installer for Oracle Cloud Infrastructure 8 | 9 | [![wercker status](https://app.wercker.com/status/7dd9fa20b980673dc0e252961950f590/s/master "wercker status")](https://app.wercker.com/project/byKey/7dd9fa20b980673dc0e252961950f590) 10 | 11 | ## About 12 | 13 | The Kubernetes Installer for Oracle Cloud Infrastructure provides a Terraform-based Kubernetes installation for Oracle 14 | Cloud Infrastructure. It consists of a set of [Terraform][terraform] modules and an example base configuration that is 15 | used to provision and configure the resources needed to run a highly available and configurable Kubernetes cluster on [Oracle Cloud Infrastructure][oci] (OCI). 16 | 17 | 18 | ## Cluster Overview 19 | 20 | Terraform is used to _provision_ the cloud infrastructure and any required local resources for the Kubernetes cluster including: 21 | 22 | #### OCI Infrastructure 23 | 24 | - Virtual Cloud Network (VCN) with dedicated subnets for etcd, masters, and workers in each availability domain 25 | - Dedicated compute instances for etcd, Kubernetes master and worker nodes in each availability domain 26 | - [Public or Private](./docs/input-variables.md#network-access-configuration) TCP/SSL OCI Load Balancer to distribute traffic to the Kubernetes Master(s) 27 | - [Public or Private](./docs/input-variables.md#network-access-configuration) TCP/SSL OCI Load Balancer to distribute traffic to the node(s) in the etcd cluster 28 | - [Optional](./docs/input-variables.md#private-network-access) NAT instance for Internet-bound traffic on any private subnets 29 | - 2048-bit SSH RSA Key-Pair for compute instances when not overridden by `ssh_private_key` and `ssh_public_key_openssh` [input variables](./docs/input-variables.md#tls-certificates--ssh-key-pair) 30 | - Self-signed CA and TLS cluster certificates when not overridden by the [input variables](./docs/input-variables.md#tls-certificates--ssh-key-pair) `ca_cert`, `ca_key`, etc. 31 | 32 | #### Cluster Configuration 33 | 34 | Terraform uses cloud-init scripts to handle the instance-level _configuration_ for instances in the Control Plane to 35 | configure: 36 | 37 | - Highly Available (HA) Kubernetes master configuration 38 | - Highly Available (HA) etcd cluster configuration 39 | - Optional [GPU support](./docs/gpu-workers.md) for worker nodes that need to run specific workloads 40 | - Kubernetes Dashboard and kube-DNS cluster add-ons 41 | - Kubernetes RBAC (role-based authorization control) 42 | - Integration with OCI [Cloud Controller Manager](https://github.com/oracle/oci-cloud-controller-manager) (CCM) 43 | - Integration with OCI [Flexvolume Driver](https://github.com/oracle/oci-flexvolume-driver) 44 | 45 | The Terraform scripts also accept a number of other [input variables](./docs/input-variables.md) to choose instance shapes (including GPU) and how they are placed across the availability domain (ADs), etc. If your requirements extend beyond the base configuration, the modules can be used to form your own customized configuration. 46 | 47 | ![](./docs/images/arch.jpg) 48 | 49 | ## Prerequisites 50 | 51 | 1. Download and install [Terraform][terraform] (v0.10.3 or later) 52 | 2. Download and install the [OCI Terraform Provider][oci provider] (v2.0.0 or later) 53 | 3. Create an Terraform configuration file at `~/.terraformrc` that specifies the path to the OCI provider: 54 | ``` 55 | providers { 56 | oci = "/terraform-provider-oci" 57 | } 58 | ``` 59 | 4. Ensure you have [Kubectl][Kubectl] installed if you plan to interact with the cluster locally 60 | 61 | ###### Optionally create separate IAM resources for OCI plugins 62 | 63 | The OCI [Cloud Controller Manager (CCM)](https://github.com/oracle/oci-cloud-controller-manager) and [Volume Provisioner (VP)](https://github.com/oracle/oci-volume-provisioner) enables Kubernetes to dynamically provision OCI resources such as Load Balancers and Block Volumes as a part of pod and service creation. In order to facilitate this, OCI credentials and OCID information are automatically stored in the cluster as a Kubernetes Secret. 64 | 65 | By default, the credentials of the user creating the cluster is used. However, in some cases, it makes sense to use a more restricted set of credentials whose policies are limited to a particular set of resources within the compartment. 66 | 67 | To Terraform separate IAM users, groups, and policy resources, run the `terraform plan` and `terraform apply` commands from the `identity` directory and set the appropriate [input variables](./docs/input-variables.md#mandatory-input-variables) relating to your custom users, fingerprints, and key paths. 68 | 69 | ## Quick start 70 | 71 | ### Customize the configuration 72 | 73 | Create a _terraform.tfvars_ file in the project root that specifies your configuration. 74 | 75 | ```bash 76 | # start from the included example 77 | $ cp terraform.example.tfvars terraform.tfvars 78 | ``` 79 | 80 | * Set [mandatory](./docs/input-variables.md#mandatory-input-variables) OCI input variables relating to your tenancy, user, and compartment. 81 | * Override [optional](./docs/input-variables.md#optional-input-variables) input variables to customize the default configuration. 82 | 83 | ### Deploy the cluster 84 | 85 | Initialize Terraform: 86 | 87 | ``` 88 | $ terraform init 89 | ``` 90 | 91 | View what Terraform plans do before actually doing it: 92 | 93 | ``` 94 | $ terraform plan 95 | ``` 96 | 97 | Use Terraform to Provision resources and stand-up k8s cluster on OCI: 98 | 99 | ``` 100 | $ terraform apply 101 | ``` 102 | 103 | ### Access the cluster 104 | 105 | The Kubernetes cluster will be running after the configuration is applied successfully and the cloud-init scripts have been given time to finish asynchronously. Typically, this takes around 5 minutes after `terraform apply` and will vary depending on the overall configuration, instance counts, and shapes. 106 | 107 | A working kubeconfig can be found in the `./generated` folder or generated on the fly using the `kubeconfig` Terraform output variable. 108 | 109 | Your network access settings determine whether your cluster is accessible from the outside. See [Accessing the Cluster](./docs/cluster-access.md) for more details. 110 | 111 | #### Verify the cluster: 112 | 113 | If you've chosen to configure a public cluster, you can do a quick and automated verification of your cluster from 114 | your local machine by running the `cluster-check.sh` located in the `scripts` directory. Note that this script requires your KUBECONFIG environment variable to be set (above), and SSH and HTTPs access to be open to etcd and worker nodes. 115 | 116 | To temporarily open access SSH and HTTPs access for `cluster-check.sh`, add the following to your `terraform.tfvars` file: 117 | 118 | ```bash 119 | # warning: 0.0.0.0/0 is wide open. remember to undo this. 120 | etcd_ssh_ingress = "0.0.0.0/0" 121 | master_ssh_ingress = "0.0.0.0/0" 122 | worker_ssh_ingress = "0.0.0.0/0" 123 | master_https_ingress = "0.0.0.0/0" 124 | worker_nodeport_ingress = "0.0.0.0/0" 125 | ``` 126 | 127 | ```bash 128 | $ scripts/cluster-check.sh 129 | ``` 130 | ``` 131 | [cluster-check.sh] Running some basic checks on Kubernetes cluster.... 132 | [cluster-check.sh] Checking ssh connectivity to each node... 133 | [cluster-check.sh] Checking whether instance bootstrap has completed on each node... 134 | [cluster-check.sh] Checking Flannel's etcd key from each node... 135 | [cluster-check.sh] Checking whether expected system services are running on each node... 136 | [cluster-check.sh] Checking status of /healthz endpoint at each k8s master node... 137 | [cluster-check.sh] Checking status of /healthz endpoint at the LB... 138 | [cluster-check.sh] Running 'kubectl get nodes' a number of times through the master LB... 139 | 140 | The Kubernetes cluster is up and appears to be healthy. 141 | Kubernetes master is running at https://129.146.22.175:443 142 | KubeDNS is running at https://129.146.22.175:443/api/v1/proxy/namespaces/kube-system/services/kube-dns 143 | kubernetes-dashboard is running at https://129.146.22.175:443/ui 144 | ``` 145 | 146 | ### Deploy a simple load-balanced application with shared volumes 147 | 148 | Check out the [example application deployment](./docs/example-deployments.md) for a walk through of deploying a simple application that leverages both the Cloud Controller Manager and Flexvolume Driver plugins. 149 | 150 | ### Scale, upgrade, or delete the cluster 151 | 152 | Check out the [example cluster operations](./docs/examples.md) for details on how to use Terraform to scale, upgrade, replace, or delete your cluster. 153 | 154 | ## Known issues and limitations 155 | 156 | * The OCI Load Balancers that gets created and attached to the VCN when a service of type `--type=LoadBalancer` is an out-of-band change to Terraform. As a result, the cluster's VCN will not be able to be destroyed until all services of type `LoadBalancer` have been deleted using `kubectl` or the OCI Console. 157 | * The OCI Block Volumes that gets created and attached to the workers when persistent volumes are create is also an out-of-band change to Terraform. As a result, the instances will not be able to be destroyed until all persistent volumes have been deleted using `kubectl` or the OCI Console. 158 | * Scaling or replacing etcd members in or out after the initial deployment is currently unsupported 159 | * Failover or HA configuration for NAT instance(s) is currently unsupported 160 | * Resizing the iSCSI volume will delete and recreate the volume 161 | * GPU Bare Metal instance shapes are currently only available in the Ashburn region and may be limited to specific availability domains 162 | * Provisioning a _mix_ of GPU-enabled and non-GPU-enabled worker node instance shapes is currently unsupported 163 | 164 | ## Testing 165 | 166 | Tests run _automatically_ on every commit to the main branch. Additionally, the tests should be run against any pull-request before it is merged. 167 | 168 | See [Testing](tests/README.md) for details. 169 | 170 | ## Contributing 171 | 172 | This project is open source. Oracle appreciates any contributions that are made by the open source community. 173 | 174 | See [Contributing](CONTRIBUTING.md) for details. 175 | -------------------------------------------------------------------------------- /bashsource.tf: -------------------------------------------------------------------------------- 1 | resource null_resource "build_source" { 2 | provisioner "local-exec" { 3 | command = "echo \"export KUBECONFIG=${path.root}/generated/kubeconfig\" > ${var.label_prefix}source.sh" 4 | } 5 | } 6 | 7 | resource null_resource "etcd-ad1" { 8 | count = "${var.etcdAd1Count}" 9 | depends_on = [ 10 | "module.instances-etcd-ad1", 11 | "null_resource.build_source" 12 | ] 13 | 14 | triggers { 15 | etcd_id = "${element(module.instances-etcd-ad1.ids, count.index)}" 16 | build_source_id = "${null_resource.build_source.id}" 17 | } 18 | 19 | provisioner "local-exec" { 20 | command = "echo 'alias ${var.label_prefix}etcdad1-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-etcd-ad1.instance_public_ips, count.index)}\"' >> source.sh" 21 | } 22 | } 23 | 24 | resource null_resource "etcd-ad2" { 25 | count = "${var.etcdAd2Count}" 26 | depends_on = [ 27 | "module.instances-etcd-ad2", 28 | "null_resource.build_source" 29 | ] 30 | 31 | triggers { 32 | etcd_id = "${element(module.instances-etcd-ad2.ids, count.index)}" 33 | build_source_id = "${null_resource.build_source.id}" 34 | } 35 | 36 | provisioner "local-exec" { 37 | command = "echo 'alias ${var.label_prefix}etcdad2-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-etcd-ad2.instance_public_ips, count.index)}\"' >> source.sh" 38 | } 39 | } 40 | 41 | 42 | resource null_resource "etcd-ad3" { 43 | count = "${var.etcdAd3Count}" 44 | depends_on = [ 45 | "module.instances-etcd-ad3", 46 | ] 47 | 48 | triggers { 49 | etcd_id = "${element(module.instances-etcd-ad3.ids, count.index)}" 50 | build_source_id = "${null_resource.build_source.id}" 51 | } 52 | 53 | provisioner "local-exec" { 54 | command = "echo 'alias ${var.label_prefix}etcad3-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-etcd-ad3.instance_public_ips, count.index)}\"' >> source.sh" 55 | } 56 | } 57 | 58 | 59 | 60 | 61 | 62 | resource null_resource "k8smaster-ad1" { 63 | count = "${var.k8sMasterAd1Count}" 64 | depends_on = [ 65 | "module.instances-k8smaster-ad1", 66 | ] 67 | 68 | triggers { 69 | master_id = "${element(module.instances-k8smaster-ad1.ids, count.index)}" 70 | build_source_id = "${null_resource.build_source.id}" 71 | } 72 | 73 | provisioner "local-exec" { 74 | command = "echo 'alias ${var.label_prefix}masterad1-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-k8smaster-ad1.public_ips, count.index)}\"' >> source.sh" 75 | } 76 | } 77 | 78 | resource null_resource "k8smaster-ad2" { 79 | count = "${var.k8sMasterAd2Count}" 80 | depends_on = [ 81 | "module.instances-k8smaster-ad2", 82 | ] 83 | 84 | triggers { 85 | master_id = "${element(module.instances-k8smaster-ad2.ids, count.index)}" 86 | build_source_id = "${null_resource.build_source.id}" 87 | } 88 | 89 | provisioner "local-exec" { 90 | command = "echo 'alias ${var.label_prefix}masterad2-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-k8smaster-ad2.public_ips, count.index)}\"' >> source.sh" 91 | } 92 | } 93 | 94 | 95 | resource null_resource "k8smaster-ad3" { 96 | count = "${var.k8sMasterAd3Count}" 97 | depends_on = [ 98 | "module.instances-k8smaster-ad3", 99 | ] 100 | 101 | triggers { 102 | master_id = "${element(module.instances-k8smaster-ad3.ids, count.index)}" 103 | build_source_id = "${null_resource.build_source.id}" 104 | } 105 | 106 | provisioner "local-exec" { 107 | command = "echo 'alias ${var.label_prefix}masterad3-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-k8smaster-ad3.public_ips, count.index)}\"' >> source.sh" 108 | } 109 | } 110 | 111 | resource null_resource "k8sworker-ad1" { 112 | count = "${var.k8sWorkerAd1Count}" 113 | depends_on = [ 114 | "module.instances-k8sworker-ad1", 115 | ] 116 | 117 | triggers { 118 | worker_id = "${element(module.instances-k8sworker-ad1.ids, count.index)}" 119 | build_source_id = "${null_resource.build_source.id}" 120 | } 121 | 122 | provisioner "local-exec" { 123 | command = "echo 'alias ${var.label_prefix}workerad1-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-k8sworker-ad1.public_ips, count.index)}\"' >> source.sh" 124 | } 125 | } 126 | 127 | resource null_resource "k8sworker-ad2" { 128 | count = "${var.k8sWorkerAd2Count}" 129 | depends_on = [ 130 | "module.instances-k8sworker-ad2", 131 | ] 132 | 133 | triggers { 134 | worker_id = "${element(module.instances-k8sworker-ad2.ids, count.index)}" 135 | build_source_id = "${null_resource.build_source.id}" 136 | } 137 | 138 | provisioner "local-exec" { 139 | command = "echo 'alias ${var.label_prefix}workerad2-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-k8sworker-ad2.public_ips, count.index)}\"' >> source.sh" 140 | } 141 | } 142 | 143 | 144 | resource null_resource "k8sworker-ad3" { 145 | count = "${var.k8sWorkerAd3Count}" 146 | depends_on = [ 147 | "module.instances-k8sworker-ad3", 148 | ] 149 | 150 | triggers { 151 | master_id = "${element(module.instances-k8sworker-ad3.ids, count.index)}" 152 | build_source_id = "${null_resource.build_source.id}" 153 | } 154 | 155 | provisioner "local-exec" { 156 | command = "echo 'alias ${var.label_prefix}workerad3-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-k8sworker-ad3.public_ips, count.index)}\"' >> source.sh" 157 | } 158 | } 159 | 160 | -------------------------------------------------------------------------------- /datasources.tf: -------------------------------------------------------------------------------- 1 | data "oci_identity_availability_domains" "ADs" { 2 | compartment_id = "${var.tenancy_ocid}" 3 | } 4 | 5 | resource "template_file" "etcd_discovery_url" { 6 | provisioner "local-exec" { 7 | command = "[ -d ${path.root}/generated ] || mkdir -p ${path.root}/generated && curl --retry 3 https://discovery.etcd.io/new?size=${var.etcdAd1Count + var.etcdAd2Count + var.etcdAd3Count} > ${path.root}/generated/discovery${self.id}" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /docs/cluster-access.md: -------------------------------------------------------------------------------- 1 | # Accessing the Cluster 2 | 3 | ## Access the cluster using kubectl, continuous build pipelines, or other clients 4 | 5 | If you've chosen to configure a _public_ Load Balancer for your Kubernetes Master(s) (i.e. `control_plane_subnet_access=public` or 6 | `control_plane_subnet_access=private` _and_ `k8s_master_lb_access=public`), you can interact with your cluster using kubectl, continuous build 7 | pipelines, or any other client over the Internet. A working kubeconfig can be found in the ./generated folder or generated on the fly using the `kubeconfig` Terraform output variable. 8 | 9 | ```bash 10 | # warning: 0.0.0.0/0 is wide open. Consider limiting HTTPs ingress to smaller set of IPs. 11 | $ terraform plan -var master_https_ingress=0.0.0.0/0 12 | $ terraform apply -var master_https_ingress=0.0.0.0/0 13 | # consider closing access off again using terraform apply -var master_https_ingress=10.0.0.0/16 14 | ``` 15 | 16 | ```bash 17 | $ export KUBECONFIG=`pwd`/generated/kubeconfig 18 | $ kubectl cluster-info 19 | $ kubectl get nodes 20 | ``` 21 | 22 | If you've chosen to configure a strictly _private_ cluster (i.e. `control_plane_subnet_access=private` _and_ `k8s_master_lb_access=private`), 23 | access to the cluster will be limited to the NAT instance(s) similar to how you would use a bastion host e.g. 24 | 25 | ```bash 26 | $ terraform plan -var public_subnet_ssh_ingress=0.0.0.0/0 27 | $ terraform apply -var public_subnet_ssh_ingress=0.0.0.0/0 28 | $ terraform output ssh_private_key > generated/instances_id_rsa 29 | $ chmod 600 generated/instances_id_rsa 30 | $ scp -i generated/instances_id_rsa generated/instances_id_rsa opc@NAT_INSTANCE_PUBLIC_IP:/home/opc/ 31 | $ ssh -i generated/instances_id_rsa opc@NAT_INSTANCE_PUBLIC_IP 32 | nat$ ssh -i /home/opc/instances_id_rsa opc@K8SMASTER_INSTANCE_PRIVATE_IP 33 | master$ kubectl cluster-info 34 | master$ kubectl get nodes 35 | ``` 36 | 37 | Note, for easier access, consider setting up an SSH tunnel between your local host and a NAT instance. 38 | 39 | ## Access the cluster using Kubernetes Dashboard 40 | 41 | Assuming `kubectl` has access to the Kubernetes Master Load Balancer, you can use `kubectl proxy` to access the 42 | Dashboard: 43 | 44 | ``` 45 | kubectl proxy & 46 | open http://localhost:8001/ui 47 | ``` 48 | 49 | ## SSH into OCI Instances 50 | 51 | If you've chosen to launch your control plane instance in _public_ subnets (i.e. `control_plane_subnet_access=public`), you can open 52 | access SSH access to your master nodes by adding the following to your `terraform.tfvars` file: 53 | 54 | ```bash 55 | # warning: 0.0.0.0/0 is wide open. remember to undo this. 56 | etcd_ssh_ingress = "0.0.0.0/0" 57 | master_ssh_ingress = "0.0.0.0/0" 58 | worker_ssh_ingress = "0.0.0.0/0" 59 | ``` 60 | 61 | ```bash 62 | # Create local SSH private key file logging into OCI instances 63 | $ terraform output ssh_private_key > generated/instances_id_rsa 64 | # Retrieve public IP for etcd nodes 65 | $ terraform output etcd_public_ips 66 | # Log in as user opc to the OEL OS 67 | $ ssh -i `pwd`/generated/instances_id_rsa opc@ETCD_INSTANCE_PUBLIC_IP 68 | # Retrieve public IP for k8s masters 69 | $ terraform output master_public_ips 70 | $ ssh -i `pwd`/generated/instances_id_rsa opc@K8SMASTER_INSTANCE_PUBLIC_IP 71 | # Retrieve public IP for k8s workers 72 | $ terraform output worker_public_ips 73 | $ ssh -i `pwd`/generated/instances_id_rsa opc@K8SWORKER_INSTANCE_PUBLIC_IP 74 | ``` 75 | 76 | If you've chosen to launch your control plane instance in _private_ subnets (i.e. `control_plane_subnet_access=private`), you'll 77 | need to first SSH into a NAT instance, then to a worker, master, or etcd node: 78 | 79 | ```bash 80 | $ terraform plan -var public_subnet_ssh_ingress=0.0.0.0/0 81 | $ terraform apply -var public_subnet_ssh_ingress=0.0.0.0/0 82 | $ terraform output ssh_private_key > generated/instances_id_rsa 83 | $ chmod 600 generated/instances_id_rsa 84 | $ terraform output nat_instance_public_ips 85 | $ scp -i generated/instances_id_rsa generated/instances_id_rsa opc@NAT_INSTANCE_PUBLIC_IP:/home/opc/ 86 | $ ssh -i generated/instances_id_rsa opc@NAT_INSTANCE_PUBLIC_IP 87 | nat$ ssh -i /home/opc/instances_id_rsa opc@PRIVATE_IP 88 | ``` 89 | -------------------------------------------------------------------------------- /docs/example-deployments.md: -------------------------------------------------------------------------------- 1 | # Example Application Deployment 2 | 3 | The following example walks through running a simple Nginx web server that leverages both the Cloud Controller Manager and Flexvolume Driver plugins through Kubernetes Services, Persistent Volumes, and Persistent Volume Claims. 4 | 5 | ### Create an dynamic OCI Block Volume using a Kubernetes PersistentVolumeClaim 6 | 7 | We'll start by creating a [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) (PVC). The cluster is integrated with the OCI [Flexvolume Driver](https://github.com/oracle/oci-flexvolume-driver). As a result, creating a PVC will result in a block storage volume to (dynamically) be created in your tenancy. 8 | 9 | Note that the matchLabels should contain the Availability Domain (AD) you want to provision a volume in, which should match the zone of at least one of your worker nodes: 10 | 11 | ```bash 12 | $ kubectl describe nodes | grep zone 13 | failure-domain.beta.kubernetes.io/zone=US-ASHBURN-AD-1 14 | failure-domain.beta.kubernetes.io/zone=US-ASHBURN-AD-2 15 | ``` 16 | 17 | ```bash 18 | $ cat nginx-pvc.yaml 19 | 20 | kind: PersistentVolumeClaim 21 | apiVersion: v1 22 | metadata: 23 | name: nginx-volume 24 | spec: 25 | storageClassName: "oci" 26 | selector: 27 | matchLabels: 28 | oci-availability-domain: "US-ASHBURN-AD-1" 29 | accessModes: 30 | - ReadWriteOnce 31 | resources: 32 | requests: 33 | storage: 50Gi 34 | ``` 35 | 36 | To add the PersistentVolumeClaim, run the following: 37 | 38 | ```bash 39 | $ kubectl apply -f nginx-pvc.yaml 40 | ``` 41 | 42 | After applying the PVC, you should see a block storage volume available in your OCI tenancy. 43 | 44 | ```bash 45 | $ kubectl get pv,pvc 46 | ``` 47 | 48 | ### Create a Kubernetes Deployment that references the PVC 49 | 50 | Now you have a PVC, you can create a Kubernetes deployment that will consume the storage: 51 | 52 | ```bash 53 | $ cat nginx-deployment.yaml 54 | 55 | apiVersion: extensions/v1beta1 56 | kind: Deployment 57 | metadata: 58 | name: nginx 59 | spec: 60 | replicas: 2 61 | template: 62 | metadata: 63 | labels: 64 | name: nginx 65 | spec: 66 | containers: 67 | - name: nginx 68 | image: nginx 69 | imagePullPolicy: IfNotPresent 70 | ports: 71 | - containerPort: 80 72 | volumeMounts: 73 | - name: nginx-storage 74 | mountPath: "/usr/share/nginx/html" 75 | volumes: 76 | - name: nginx-storage 77 | persistentVolumeClaim: 78 | claimName: nginx-volume 79 | ``` 80 | 81 | To run the deployment, run the following: 82 | 83 | ```bash 84 | $ kubectl apply -f nginx-deployment.yaml 85 | ``` 86 | 87 | After applying the change, your pods should be scheduled on nodes running in the same AD of your volume and all have access to the shared volume: 88 | 89 | ``` 90 | $ kubectl get pods -o wide 91 | NAME READY STATUS RESTARTS AGE IP NODE 92 | nginx-r1 1/1 Running 0 35s 10.99.46.4 k8s-worker-ad1-0.k8sworkerad1.k8soci.oraclevcn.com 93 | nginx-r2 1/1 Running 0 35s 10.99.46.5 k8s-worker-ad1-0.k8sworkerad1.k8soci.oraclevcn.com 94 | ``` 95 | 96 | ``` 97 | $ kubectl exec nginx-r1 touch /usr/share/nginx/html/test 98 | ``` 99 | 100 | ``` 101 | $ kubectl exec nginx-r2 ls /usr/share/nginx/html 102 | test 103 | lost+found 104 | ``` 105 | 106 | ### Expose the app using the Cloud Controller Manager 107 | 108 | The cluster is integrated with the OCI [Cloud Controller Manager](https://github.com/oracle/oci-cloud-controller-manager) (CCM). As a result, creating a service of type `--type=LoadBalancer` will expose the pods to the Internet using an OCI Load Balancer. 109 | 110 | ```bash 111 | $ kubectl expose deployment nginx --port=80 --type=LoadBalancer 112 | ``` 113 | 114 | List service to get the external IP address (OCI LoadBalancer) of your exposed service. Note, the IP will be listed as `` while the load balancer is being provisioned: 115 | 116 | ```bash 117 | $ kubectl get service nginx 118 | ``` 119 | 120 | Access the Nginx service 121 | 122 | ``` 123 | open http://:80 124 | ``` 125 | 126 | ### Clean up 127 | 128 | Clean up the container, OCI Load Balancer, and Block Volume by deleting the deployment, service, and persistent volume claim: 129 | 130 | ```bash 131 | $ kubectl delete service nginx 132 | ``` 133 | 134 | ```bash 135 | $ kubectl delete -f nginx-deployment.yaml 136 | ``` 137 | 138 | ```bash 139 | $ kubectl delete -f nginx-pvc.yaml 140 | ``` 141 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Example Installer Operations 2 | 3 | ## Deploying a new cluster using terraform apply 4 | 5 | Override any of the above input variables in your terraform.vars and run the plan and apply commands: 6 | 7 | ```bash 8 | # verify what will change 9 | $ terraform plan 10 | 11 | # scale workers 12 | $ terraform apply 13 | ``` 14 | 15 | ## Scaling k8s workers (in or out) using terraform apply 16 | 17 | To scale workers in or out, adjust the `k8sWorkerAd1Count`, `k8sWorkerAd2Count`, or `k8sWorkerAd3Count` input 18 | variables in terraform.vars and run the plan and apply commands: 19 | 20 | ```bash 21 | # verify changes 22 | $ terraform plan 23 | 24 | # scale workers (use -target=module.instances-k8sworker-adX to only target workers in a particular AD) 25 | $ terraform apply 26 | ``` 27 | 28 | When scaling worker nodes _up_, you will need to wait for the node initialization to finish asynchronously before 29 | the new nodes will be seen with `kubectl get nodes` 30 | 31 | When scaling worker nodes _down_, the instances/k8sworker module's user_data code will take care of running `kubectl drain` and `kubectl delete node` on the nodes being terminated. 32 | 33 | ## Scaling k8s masters (in or out) using terraform apply 34 | 35 | To scale the masters in or out, adjust the `k8sMasterAd1Count`, `k8sMasterAd2Count`, or `k8sMasterAd3Count` input variables in terraform.vars and run the plan and apply commands: 36 | 37 | ```bash 38 | # verify changes 39 | $ terraform plan 40 | 41 | # scale master nodes 42 | $ terraform apply 43 | ``` 44 | 45 | Similar to the initial deployment, you will need to wait for the node initialization to finish asynchronously. 46 | 47 | ## Scaling etcd nodes (in or out) using terraform apply 48 | 49 | Scaling the etcd nodes in or out after the initial deployment is not currently supported. Terminating all the nodes in the etcd cluster will result in data loss. 50 | 51 | ## Replacing worker nodes using terraform taint 52 | 53 | We can use `terraform taint` to worker instances in a particular AD as "tainted", which will cause 54 | them to be drained, destroyed, and recreated on the next apply. This can be a useful strategy for reverting local changes or 55 | regenerating a misbehaving worker. 56 | 57 | ```bash 58 | # taint all workers in a particular AD 59 | terraform taint -module=instances-k8sworker-ad1 oci_core_instance.TFInstanceK8sWorker 60 | # optionally taint workers in AD2 and AD3 or do so in a subsequent apply 61 | # terraform taint -module=instances-k8sworker-ad2 oci_core_instance.TFInstanceK8sWorker 62 | # terraform taint -module=instances-k8sworker-ad3 oci_core_instance.TFInstanceK8sWorker 63 | 64 | # preview changes 65 | $ terraform plan 66 | 67 | # drain and replace workers 68 | $ terraform apply 69 | ``` 70 | 71 | When you are ready to make the new worker node schedulable again, use `kubectl uncordon` to undo the `kubectl drain`. 72 | 73 | ## Replacing masters using terraform taint 74 | 75 | We can also use `terraform taint` to master instances in a particular AD as "tainted", which will cause 76 | them to be destroyed and recreated on the next apply. This can be a useful strategy for reverting local 77 | changes or regenerating a misbehaving master. 78 | 79 | ```bash 80 | # taint all masters in a particular AD 81 | terraform taint -module=instances-k8smaster-ad1 oci_core_instance.TFInstanceK8sMaster 82 | # optionally taint masters in AD2 and AD3 or do so in a subsequent apply 83 | # terraform taint -module=instances-k8smaster-ad2 oci_core_instance.TFInstanceK8sMaster 84 | # terraform taint -module=instances-k8smaster-ad3 oci_core_instance.TFInstanceK8sMaster 85 | 86 | # preview changes 87 | $ terraform plan 88 | 89 | # replace workers 90 | $ terraform apply 91 | ``` 92 | 93 | ## Upgrading Kubernetes Version 94 | 95 | There are a few ways of moving to a new version of Kubernetes in your cluster. 96 | 97 | The easiest way to upgrade to a new Kubernetes version is to use the scripts to do a fresh cluster install using an updated `k8s_ver` inpput variable. The downside with this option is that the new cluster will not have your existing cluster state and deployments. 98 | 99 | The other options involve using the `k8s_ver` input variable to _replace_ master and worker instances in your _existing_ cluster. We can replace master and worker instances in the cluster since Kubernetes masters and workers are stateless. This option can either be done all at once or incrementally. 100 | 101 | #### Option 1: Do a clean install (easiest overall approach) 102 | 103 | Set the `k8s_ver` and follow the original instructions in the [README](../README.md) do install a new cluster. The `label_prefix` variable is useful for installing multiple clusters in a compartment. 104 | 105 | #### Option 2: Upgrade cluster all at once (easiest upgrade) 106 | 107 | The example `terraform apply` command below will destroy then re-create all master and worker instances using as much parallelism as possible. It's the easiest and quickest upgrade scenario, but will result in some downtime for the workers and masters while they are being re-created. The single example `terraform apply` below will: 108 | 109 | 1. drain, destroy all worker nodes 110 | 2. destroy all master nodes 111 | 3. destroy all master load-balancer backends that point to old master instances 112 | 4. re-create master instances using Kubernetes 1.7.5 113 | 5. re-create worker nodes using Kubernetes 1.7.5 114 | 6. re-create master load-balancer backends to point to new master node instances 115 | 116 | ```bash 117 | # preview upgrade/replace 118 | $ terraform plan -var k8s_ver=1.7.5 119 | 120 | # perform upgrade/replace 121 | $ terraform apply -var k8s_ver=1.7.5 122 | ``` 123 | 124 | When you are ready to make the new 1.7.5 worker node schedulable, use `kubectl uncordon`. 125 | 126 | #### Option 3: Upgrade cluster instances incrementally (most complicated, most control over roll-out) 127 | 128 | ##### First, upgrade master nodes by AD 129 | 130 | If you would rather update the cluster incrementally, we start by upgrading the master nodes in each AD. In this scenario, each `terraform apply` will: 131 | 132 | 1. destroy all master instances in a particular AD 133 | 2. destroy all master load-balancer backends that point to deleted master instances 134 | 3. re-create master instances in the AD using Kubernetes 1.7.5 135 | 4. re-create master load-balancer backends to point to new master node instances 136 | 137 | For example, here is the command to upgrade all the master instances in AD1: 138 | 139 | ```bash 140 | # preview upgrade of all masters and their LB backends in AD1 141 | $ terraform plan -var k8s_ver=1.7.5 -target=module.instances-k8smaster-ad1 -target=module.k8smaster-public-lb 142 | 143 | # perform upgrade/replace masters 144 | $ terraform apply -var k8s_ver=1.7.5 -target=module.instances-k8smaster-ad1 -target=module.k8smaster-public-lb 145 | ``` 146 | 147 | Be sure to repeat this command for each AD you have masters on. 148 | 149 | ##### Next, upgrade worker nodes by AD 150 | 151 | After upgrading all the master nodes, we upgrade the worker nodes in each AD. Each `terraform apply` will: 152 | 153 | 1. drain all worker nodes in a particular AD to your nodes in AD2 and AD3 154 | 2. destroy all worker nodes in a particular AD 155 | 3. re-create worker nodes in a particular AD using Kubernetes 1.7.5 156 | 157 | For example, here is the command to upgrade the master instances in AD1: 158 | 159 | ```bash 160 | # preview upgrade of all workers in a particular AD to K8s 161 | $ terraform plan -var k8s_ver=1.7.5 -target=module.instances-k8sworker-ad1 162 | 163 | # perform upgrade/replace workers 164 | $ terraform apply -var k8s_ver=1.7.5 -target=module.instances-k8sworker-ad1 165 | ``` 166 | 167 | Like before, repeat `terraform apply` on each AD you have workers on. Note that if you have more than one worker in an AD, you can upgrade worker nodes individually using the subscript operator e.g. 168 | 169 | ```bash 170 | # preview upgrade of a single worker in a particular AD to K8s 1.7.5 171 | $ terraform plan -var k8s_ver=1.7.5 -target=module.instances-k8smaster-ad1.oci_core_instance.TFInstanceK8sMaster[1] 172 | 173 | # perform upgrade/replace of worker 174 | $ terraform apply -var k8s_ver=1.7.5 -target=module.instances-k8sworker-ad1 175 | ``` 176 | 177 | When you are ready to make the new 1.7.5 worker node schedulable, use `kubectl uncordon`. 178 | 179 | ## Replacing etcd cluster members using terraform taint 180 | 181 | Replacing etcd cluster members after the initial deployment is not currently supported. 182 | 183 | ## Deploying a GPU-enabled cluster 184 | 185 | See [deploying GPU-enabled worker nodes](./gpu-workers.md) for details. 186 | 187 | ## Deleting a cluster using terraform destroy 188 | 189 | Don't forget to delete any OCI Load Balancers that were created by the [Cloud Controller Manager](https://github.com/oracle/oci-cloud-controller-manager) for services with `--type=LoadBalancer` by running `kubectl delete svc` before trying to destroy the cluster using Terraform. 190 | 191 | ```bash 192 | $ terraform destroy 193 | ``` -------------------------------------------------------------------------------- /docs/gpu-workers.md: -------------------------------------------------------------------------------- 1 | # GPUs on Worker Nodes 2 | 3 | Oracle Cloud Infrastructure provides GPU instance shapes and corresponding OS images that have the required drivers and libraries, which can be used to run specific workloads. 4 | 5 | ## Deploying a cluster with GPU-enabled worker nodes 6 | 7 | To enable GPU on your worker nodes, set the GPU instance shapes and OS input variables in your in terraform.vars e.g.: `k8sWorkerShape = "BM.GPU2.2"` and `worker_ol_image_name = "Oracle-Linux-7.4-Gen2-GPU-2018.01.10-0"`. 8 | 9 | Next, run the plan and apply commands: 10 | 11 | ```bash 12 | # verify changes 13 | $ terraform plan 14 | 15 | # Note, GPU instance shapes are currently only available in the Ashburn region 16 | $ terraform apply 17 | ``` 18 | 19 | The Kubernetes worker nodes nodes will have both GPU devices and the NVIDIA drivers pre-installed and configured. The kubelet will be pre-configured with the `Accelerators=true` option which enables the alpha.kubernetes.io/nvidia-gpu as a schedulable resource: 20 | 21 | ``` 22 | $ kubectl describe nodes 23 | ... 24 | Capacity: 25 | alpha.kubernetes.io/nvidia-gpu: 2 26 | cpu: 56 27 | memory: 196439168Ki 28 | pods: 110 29 | ``` 30 | 31 | 32 | ## Assign GPU resources to pods 33 | 34 | You can request and consume the GPUs on your worker nodes from pods by requesting `alpha.kubernetes.io/nvidia-gpu` resources as you would CPU or memory. Note that the NVIDIA and CUDA libraries that are installed on the worker needs to also be available to the pod. So, we are mounting them as volumes in these examples. 35 | 36 | Here's a simple example pod requesting 1 GPU, and running the NVIDIA System Management Interface (nvidia-smi) command line utility to list the available GPUs and exit: 37 | 38 | ```bash 39 | $ cat nvidia-smi.yaml 40 | 41 | apiVersion: v1 42 | kind: Pod 43 | metadata: 44 | name: nvidia-smi 45 | spec: 46 | containers: 47 | - image: nvidia/cuda 48 | name: nvidia-smi 49 | command: [ "nvidia-smi" ] 50 | resources: 51 | limits: 52 | alpha.kubernetes.io/nvidia-gpu: 1 53 | requests: 54 | alpha.kubernetes.io/nvidia-gpu: 1 55 | volumeMounts: 56 | - mountPath: /usr/bin/ 57 | name: binaries 58 | - mountPath: /usr/lib64/nvidia 59 | name: libraries 60 | restartPolicy: Never 61 | volumes: 62 | - name: binaries 63 | hostPath: 64 | path: /bin/ 65 | - name: libraries 66 | hostPath: 67 | path: /usr/lib64/nvidia 68 | ``` 69 | 70 | Deploy and print the pod logs: 71 | 72 | ``` 73 | kubectl apply -f nvidia-smi.yaml 74 | kubectl get pods --show-all 75 | kubectl logs nvidia-smi 76 | ``` 77 | 78 | Here's an example of a pod requesting 2 GPUs, and importing the tensorflow GPUs and exits: 79 | 80 | ```bash 81 | $ cat tenserflow.yaml 82 | 83 | kind: Pod 84 | apiVersion: v1 85 | metadata: 86 | name: nvidia-tensorflow 87 | spec: 88 | containers: 89 | - name: gpu-container 90 | image: tensorflow/tensorflow:latest-gpu 91 | imagePullPolicy: Always 92 | command: ["python", "-c", "import tensorflow as tf; print(tf.__version__)"] 93 | resources: 94 | requests: 95 | alpha.kubernetes.io/nvidia-gpu: 2 96 | limits: 97 | alpha.kubernetes.io/nvidia-gpu: 2 98 | volumeMounts: 99 | - name: libraries 100 | mountPath: /usr/local/nvidia/lib64 101 | readOnly: true 102 | restartPolicy: Never 103 | volumes: 104 | - name: libraries 105 | hostPath: 106 | path: /usr/lib64/nvidia 107 | ``` 108 | 109 | Deploy and print the pod logs: 110 | 111 | ``` 112 | kubectl apply -f tenserflow.yaml 113 | kubectl get pods --show-all 114 | kubectl logs nvidia-tensorflow 115 | ``` 116 | 117 | ## Known issues and limitations 118 | 119 | * Scheduling GPUs in Kubernetes is still experimental, but is enabled by default in the Kubernetes Installer for OCI. 120 | * GPU Bare Metal instance shapes are currently only available in the Ashburn region and may be limited to specific availability domains. 121 | * The Kubernetes Installer for OCI does not support provisioning a _mix_ of GPU-enabled and non-GPU-enabled worker node instance shapes. The cluster either has _all_ or _no_ GPU-enabled workers. 122 | -------------------------------------------------------------------------------- /docs/images/arch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/terraform-kubernetes-installer/3a4952f2cb605e77d02c08484d42752c1bcc18fa/docs/images/arch.jpg -------------------------------------------------------------------------------- /docs/images/private_cp_subnet_private_lb_access.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/terraform-kubernetes-installer/3a4952f2cb605e77d02c08484d42752c1bcc18fa/docs/images/private_cp_subnet_private_lb_access.jpg -------------------------------------------------------------------------------- /docs/images/private_cp_subnet_public_lb_access.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/terraform-kubernetes-installer/3a4952f2cb605e77d02c08484d42752c1bcc18fa/docs/images/private_cp_subnet_public_lb_access.jpg -------------------------------------------------------------------------------- /docs/images/private_cp_subnet_public_lb_failure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/terraform-kubernetes-installer/3a4952f2cb605e77d02c08484d42752c1bcc18fa/docs/images/private_cp_subnet_public_lb_failure.jpg -------------------------------------------------------------------------------- /docs/images/public_cp_subnet_access.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oracle/terraform-kubernetes-installer/3a4952f2cb605e77d02c08484d42752c1bcc18fa/docs/images/public_cp_subnet_access.jpg -------------------------------------------------------------------------------- /identity/cloud_controller_user.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "cloud_controller_user_key" { 2 | algorithm = "RSA" rsa_bits = 2048 3 | } 4 | 5 | resource "oci_identity_group" "cloud_controller_group" { 6 | name = "${var.label_prefix}cloud_controller_group" 7 | description = "Terraform created group for OCI Cloud Controller Manager" 8 | } 9 | 10 | resource "oci_identity_user" "cloud_controller_user" { 11 | name = "${var.label_prefix}cloud_controller_user" 12 | description = "Terraform created user for OCI Cloud Controller Manager" 13 | } 14 | 15 | resource "oci_identity_api_key" "cloud_controller_key_assoc" { 16 | user_id = "${oci_identity_user.cloud_controller_user.id}" 17 | key_value = "${tls_private_key.cloud_controller_user_key.public_key_pem}" 18 | } 19 | 20 | resource "oci_identity_user_group_membership" "cloud_controller_user_group_assoc" { 21 | compartment_id = "${var.tenancy_ocid}" 22 | user_id = "${oci_identity_user.cloud_controller_user.id}" 23 | group_id = "${oci_identity_group.cloud_controller_group.id}" 24 | } 25 | 26 | resource "oci_identity_policy" "cloud_controller_policy" { 27 | depends_on = ["oci_identity_group.cloud_controller_group"] 28 | compartment_id = "${var.compartment_ocid}" 29 | name = "${var.label_prefix}cloud_controller_policy" 30 | description = "${var.label_prefix}cloud_controller_group policy" 31 | statements = [ 32 | "Allow group id ${oci_identity_group.cloud_controller_group.id} to manage load-balancers in compartment id ${var.compartment_ocid}", 33 | "Allow group id ${oci_identity_group.cloud_controller_group.id} to use security-lists in compartment id ${var.compartment_ocid}", 34 | "Allow group id ${oci_identity_group.cloud_controller_group.id} to read instances in compartment id ${var.compartment_ocid}", 35 | "Allow group id ${oci_identity_group.cloud_controller_group.id} to read subnets in compartment id ${var.compartment_ocid}", 36 | "Allow group id ${oci_identity_group.cloud_controller_group.id} to read vnics in compartment id ${var.compartment_ocid}", 37 | "Allow group id ${oci_identity_group.cloud_controller_group.id} to read vnic-attachments in compartment id ${var.compartment_ocid}", 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /identity/flexvolume_user.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "flexvolume_driver_user_key" { 2 | algorithm = "RSA" rsa_bits = 2048 3 | } 4 | 5 | resource "oci_identity_group" "flexvolume_driver_group" { 6 | name = "${var.label_prefix}flexvolume_driver_group" 7 | description = "Terraform created group for OCI Cloud Controller Manager" 8 | } 9 | 10 | resource "oci_identity_user" "flexvolume_driver_user" { 11 | name = "${var.label_prefix}flexvolume_driver_user" 12 | description = "Terraform created user for OCI Cloud Controller Manager" 13 | } 14 | 15 | resource "oci_identity_api_key" "flexvolume_driver_key_assoc" { 16 | user_id = "${oci_identity_user.flexvolume_driver_user.id}" 17 | key_value = "${tls_private_key.flexvolume_driver_user_key.public_key_pem}" 18 | } 19 | 20 | resource "oci_identity_user_group_membership" "flexvolume_driver_user_group_assoc" { 21 | compartment_id = "${var.tenancy_ocid}" 22 | user_id = "${oci_identity_user.flexvolume_driver_user.id}" 23 | group_id = "${oci_identity_group.flexvolume_driver_group.id}" 24 | } 25 | 26 | resource "oci_identity_policy" "flexvolume_driver_policy" { 27 | depends_on = ["oci_identity_group.flexvolume_driver_group"] 28 | compartment_id = "${var.compartment_ocid}" 29 | name = "${var.label_prefix}flexvolume_driver_policy" 30 | description = "${var.label_prefix}flexvolume_driver_group policy" 31 | statements = [ 32 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read vnic-attachments in compartment id ${var.compartment_ocid}", 33 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read vnics in compartment id ${var.compartment_ocid}", 34 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read instances in compartment id ${var.compartment_ocid}", 35 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read subnets in compartment id ${var.compartment_ocid}", 36 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to use volumes in compartment id ${var.compartment_ocid}", 37 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to use instances in compartment id ${var.compartment_ocid}", 38 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to manage volume-attachments in compartment id ${var.compartment_ocid}", 39 | 40 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read file-systems in compartment id ${var.compartment_ocid}", 41 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read mount-targets in compartment id ${var.compartment_ocid}", 42 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to read private-ips in compartment id ${var.compartment_ocid}", 43 | "Allow group id ${oci_identity_group.flexvolume_driver_group.id} to manage export-sets in compartment id ${var.compartment_ocid}", 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /identity/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_controller_user" { 2 | value = "${oci_identity_user.cloud_controller_user.id}" 3 | } 4 | 5 | output "cloud_controller_public_key" { 6 | value = "${tls_private_key.cloud_controller_user_key.public_key_pem}" 7 | } 8 | 9 | output "cloud_controller_private_key" { 10 | sensitive = true 11 | value = "${tls_private_key.cloud_controller_user_key.private_key_pem}" 12 | } 13 | 14 | output "cloud_controller_user_fingerprint" { 15 | value = "run 'terraform output cloud_controller_private_key > cc_key && openssl rsa -in cc_key -pubout -outform DER | openssl md5 -c && rm cc_key' determine the fingerprint" 16 | } 17 | 18 | output "flexvolume_driver_user" { 19 | value = "${oci_identity_user.flexvolume_driver_user.id}" 20 | } 21 | 22 | output "flexvolume_driver_public_key" { 23 | value = "${tls_private_key.flexvolume_driver_user_key.public_key_pem}" 24 | } 25 | 26 | output "flexvolume_driver_private_key" { 27 | sensitive = true 28 | value = "${tls_private_key.flexvolume_driver_user_key.private_key_pem}" 29 | } 30 | 31 | output "flexvolume_driver_user_fingerprint" { 32 | value = "run 'terraform output flexvolume_driver_private_key > cc_key && openssl rsa -in cc_key -pubout -outform DER | openssl md5 -c && rm cc_key' determine the fingerprint" 33 | } 34 | 35 | output "volume_provisioner_user" { 36 | value = "${oci_identity_user.volume_provisioner_user.id}" 37 | } 38 | 39 | output "volume_provisioner_public_key" { 40 | value = "${tls_private_key.volume_provisioner_user_key.public_key_pem}" 41 | } 42 | 43 | output "volume_provisioner_private_key" { 44 | sensitive = true 45 | value = "${tls_private_key.volume_provisioner_user_key.private_key_pem}" 46 | } 47 | 48 | output "volume_provisioner_user_fingerprint" { 49 | value = "run 'terraform output volume_provisioner_private_key > cc_key && openssl rsa -in cc_key -pubout -outform DER | openssl md5 -c && rm cc_key' determine the fingerprint" 50 | } 51 | 52 | -------------------------------------------------------------------------------- /identity/provider.tf: -------------------------------------------------------------------------------- 1 | provider "oci" { 2 | tenancy_ocid = "${var.tenancy_ocid}" 3 | user_ocid = "${var.user_ocid}" 4 | fingerprint = "${var.fingerprint}" 5 | private_key_path = "${var.private_key_path}" 6 | private_key_password = "${var.private_key_password}" 7 | region = "${var.region}" 8 | disable_auto_retries = "false" 9 | } 10 | -------------------------------------------------------------------------------- /identity/variables.tf: -------------------------------------------------------------------------------- 1 | variable "label_prefix" { 2 | description = "To create unique identifier for multiple clusters in a compartment." 3 | type = "string" 4 | } 5 | 6 | variable "compartment_ocid" {} 7 | 8 | 9 | variable "tenancy_ocid" {} 10 | variable "user_ocid" {} 11 | variable "fingerprint" {} 12 | variable "private_key_path" {} 13 | 14 | variable "private_key_password" { 15 | default = "" 16 | } 17 | 18 | variable "region" { 19 | default = "us-phoenix-1" 20 | } 21 | -------------------------------------------------------------------------------- /identity/volume_provisioner_user.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "volume_provisioner_user_key" { 2 | algorithm = "RSA" rsa_bits = 2048 3 | } 4 | 5 | resource "oci_identity_group" "volume_provisioner_group" { 6 | name = "${var.label_prefix}volume_provisioner_group" 7 | description = "Terraform created group for OCI Cloud Controller Manager" 8 | } 9 | 10 | resource "oci_identity_user" "volume_provisioner_user" { 11 | name = "${var.label_prefix}volume_provisioner_user" 12 | description = "Terraform created user for OCI Cloud Controller Manager" 13 | } 14 | 15 | resource "oci_identity_api_key" "volume_provisioner_key_assoc" { 16 | user_id = "${oci_identity_user.volume_provisioner_user.id}" 17 | key_value = "${tls_private_key.volume_provisioner_user_key.public_key_pem}" 18 | } 19 | 20 | resource "oci_identity_user_group_membership" "volume_provisioner_user_group_assoc" { 21 | compartment_id = "${var.tenancy_ocid}" 22 | user_id = "${oci_identity_user.volume_provisioner_user.id}" 23 | group_id = "${oci_identity_group.volume_provisioner_group.id}" 24 | } 25 | 26 | resource "oci_identity_policy" "volume_provisioner_policy" { 27 | depends_on = ["oci_identity_group.volume_provisioner_group"] 28 | compartment_id = "${var.compartment_ocid}" 29 | name = "${var.label_prefix}volume_provisioner_policy" 30 | description = "${var.label_prefix}volume_provisioner_group policy" 31 | statements = [ 32 | "Allow group id ${oci_identity_group.volume_provisioner_group.id} to manage volumes in compartment id ${var.compartment_ocid}", 33 | "Allow group id ${oci_identity_group.volume_provisioner_group.id} to manage file-systems in compartment id ${var.compartment_ocid}", 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /instances/etcd/cloud_init/bootstrap.template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # Turn off SELinux 4 | sudo sed -i s/SELINUX=enforcing/SELINUX=permissive/ /etc/selinux/config 5 | setenforce 0 6 | 7 | # Set working dir 8 | cd /home/opc 9 | 10 | # enable ol7 addons 11 | yum-config-manager --disable ol7_UEKR3 12 | yum-config-manager --enable ol7_addons ol7_latest ol7_UEKR4 ol7_optional ol7_optional_latest 13 | 14 | # Install Docker 15 | until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done 16 | 17 | cat < /etc/sysconfig/docker 18 | OPTIONS="--selinux-enabled --log-opt max-size=${docker_max_log_size} --log-opt max-file=${docker_max_log_files}" 19 | DOCKER_CERT_PATH=/etc/docker 20 | GOTRACEBACK=crash 21 | EOF 22 | 23 | # Start Docker 24 | systemctl daemon-reload 25 | systemctl enable docker 26 | systemctl restart docker 27 | 28 | docker info 29 | 30 | ################### 31 | # Drop firewall rules 32 | iptables -F 33 | 34 | ################### 35 | # etcd 36 | 37 | # Get IP Address of self 38 | IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip addr show | grep -Po 'inet \K[\d.]+') 39 | SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2) 40 | 41 | HOSTNAME=$(hostname) 42 | FQDN_HOSTNAME="$(getent hosts $IP_LOCAL | awk '{print $2}')" 43 | 44 | ## Login iSCSI volume mount and create filesystem at /etcd 45 | ###################################### 46 | iqn=$(iscsiadm --mode discoverydb --type sendtargets --portal 169.254.2.2:3260 --discover| cut -f2 -d" ") 47 | 48 | if [ -n "$${iqn}" ]; then 49 | echo "iSCSI Login $${iqn}" 50 | iscsiadm -m node -o new -T $${iqn} -p 169.254.2.2:3260 51 | iscsiadm -m node -o update -T $${iqn} -n node.startup -v automatic 52 | iscsiadm -m node -T $${iqn} -p 169.254.2.2:3260 -l 53 | # Wait for device to apear... 54 | until [[ -e "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" ]]; do sleep 1 && echo -n "."; done 55 | # If the volume has been created and formatted before but it's just a new instance this may fail 56 | # but if so ignore and carry on. 57 | mkfs -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1"; 58 | echo "$$(readlink -f /dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1) /etcd xfs defaults,noatime,_netdev 0 2" >> /etc/fstab 59 | mkdir -p /etcd 60 | mount -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" /etcd 61 | fi 62 | 63 | docker run -d \ 64 | --restart=always \ 65 | -p 2380:2380 -p 2379:2379 \ 66 | -v /etc/ssl/certs/ca-bundle.crt:/etc/ssl/certs/ca-bundle.crt \ 67 | -v /etcd:/$HOSTNAME.etcd \ 68 | --net=host \ 69 | quay.io/coreos/etcd:${etcd_ver} \ 70 | /usr/local/bin/etcd \ 71 | -name $HOSTNAME \ 72 | -advertise-client-urls http://$IP_LOCAL:2379 \ 73 | -listen-client-urls http://$IP_LOCAL:2379,http://127.0.0.1:2379 \ 74 | -listen-peer-urls http://0.0.0.0:2380 \ 75 | -discovery ${etcd_discovery_url} 76 | 77 | # wait for etcd to become active 78 | while ! curl -sf -o /dev/null http://$FQDN_HOSTNAME:2379/v2/keys/; do 79 | sleep 1 80 | echo "Try again" 81 | done 82 | 83 | # Download etcdctl client etcd_ver 84 | while ! curl -L https://github.com/coreos/etcd/releases/download/${etcd_ver}/etcd-${etcd_ver}-linux-amd64.tar.gz -o /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz; do 85 | sleep 1 86 | ((secs++)) && ((secs==10)) && break 87 | echo "Try again" 88 | done 89 | tar zxf /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz -C /tmp/ && cp /tmp/etcd-${etcd_ver}-linux-amd64/etcd* /usr/local/bin/ 90 | -------------------------------------------------------------------------------- /instances/etcd/datasources.tf: -------------------------------------------------------------------------------- 1 | # Prevent oci_core_images image list from changing underneath us. 2 | data "oci_core_images" "ImageOCID" { 3 | compartment_id = "${var.compartment_ocid}" 4 | display_name = "${var.oracle_linux_image_name}" 5 | } 6 | 7 | # Cloud call to get a list of Availability Domains 8 | data "oci_identity_availability_domains" "ADs" { 9 | compartment_id = "${var.tenancy_ocid}" 10 | } 11 | 12 | # "cloud init" file to bootstrap instance 13 | data "template_file" "etcd-bootstrap" { 14 | template = "${file("${path.module}/cloud_init/bootstrap.template.sh")}" 15 | 16 | vars { 17 | domain_name = "${var.domain_name}" 18 | docker_ver = "${var.docker_ver}" 19 | etcd_ver = "${var.etcd_ver}" 20 | docker_max_log_size = "${var.etcd_docker_max_log_size}" 21 | docker_max_log_files = "${var.etcd_docker_max_log_files}" 22 | etcd_discovery_url = "${file("${path.root}/generated/discovery${var.etcd_discovery_url}")}" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /instances/etcd/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * The instances/etcd module provisions and configures one or more etcd instances. 3 | */ 4 | 5 | resource "oci_core_instance" "TFInstanceEtcd" { 6 | count = "${var.count}" 7 | availability_domain = "${var.availability_domain}" 8 | compartment_id = "${var.compartment_ocid}" 9 | display_name = "${var.label_prefix}${var.display_name_prefix}-${count.index}" 10 | hostname_label = "${var.hostname_label_prefix}-${count.index}" 11 | image = "${lookup(data.oci_core_images.ImageOCID.images[0], "id")}" 12 | shape = "${var.shape}" 13 | 14 | create_vnic_details { 15 | subnet_id = "${var.subnet_id}" 16 | display_name = "${var.label_prefix}${var.display_name_prefix}-${count.index}" 17 | hostname_label = "${var.hostname_label_prefix}-${count.index}" 18 | assign_public_ip = "${(var.control_plane_subnet_access == "private") ? "false" : "true"}" 19 | private_ip = "${var.assign_private_ip == "true" ? cidrhost(lookup(var.network_cidrs,var.subnet_name), count.index+2) : ""}" 20 | }, 21 | 22 | extended_metadata { 23 | roles = "etcd" 24 | ssh_authorized_keys = "${var.ssh_public_key_openssh}" 25 | 26 | # Automate etcd instance configuration with cloud init run at launch time 27 | user_data = "${base64encode(data.template_file.etcd-bootstrap.rendered)}" 28 | tags = "group:etcd" 29 | } 30 | 31 | timeouts { 32 | create = "60m" 33 | } 34 | 35 | provisioner "local-exec" { 36 | command = "sleep 10" 37 | } 38 | } 39 | 40 | resource "oci_core_volume" "TFVolumeInstanceEtcd" { 41 | count = "${var.etcd_iscsi_volume_create ? var.count : 0}" 42 | availability_domain = "${var.availability_domain}" 43 | compartment_id = "${var.compartment_ocid}" 44 | display_name = "block-volume-${var.hostname_label_prefix}-${count.index}" 45 | size_in_gbs = "${var.etcd_iscsi_volume_size}" 46 | } 47 | 48 | resource "oci_core_volume_attachment" "TFVolumeAttachmentInstanceEtcd" { 49 | count = "${var.etcd_iscsi_volume_create ? var.count : 0}" 50 | attachment_type = "iscsi" 51 | compartment_id = "${var.compartment_ocid}" 52 | instance_id = "${oci_core_instance.TFInstanceEtcd.*.id[count.index]}" 53 | volume_id = "${oci_core_volume.TFVolumeInstanceEtcd.*.id[count.index]}" 54 | depends_on = ["oci_core_instance.TFInstanceEtcd", "oci_core_volume.TFVolumeInstanceEtcd"] 55 | } 56 | -------------------------------------------------------------------------------- /instances/etcd/outputs.tf: -------------------------------------------------------------------------------- 1 | # Output the private and public IPs of the instance 2 | 3 | output "ids" { 4 | value = ["${oci_core_instance.TFInstanceEtcd.*.id}"] 5 | } 6 | 7 | output "hostname_label" { 8 | value = "${oci_core_instance.TFInstanceEtcd.*.hostname_label}" 9 | } 10 | 11 | output "private_ips" { 12 | value = ["${oci_core_instance.TFInstanceEtcd.*.private_ip}"] 13 | } 14 | 15 | output "instance_public_ips" { 16 | value = ["${oci_core_instance.TFInstanceEtcd.*.public_ip}"] 17 | } 18 | -------------------------------------------------------------------------------- /instances/etcd/variables.tf: -------------------------------------------------------------------------------- 1 | variable "network_cidrs" { 2 | type = "map" 3 | } 4 | variable "availability_domain" {} 5 | variable "compartment_ocid" {} 6 | variable "display_name_prefix" {} 7 | variable "hostname_label_prefix" {} 8 | 9 | variable "shape" { 10 | default = "VM.Standard1.1" 11 | } 12 | 13 | variable "subnet_id" {} 14 | variable "subnet_name" {} 15 | variable "ssh_public_key_openssh" {} 16 | variable "domain_name" {} 17 | 18 | variable "label_prefix" { 19 | default = "" 20 | } 21 | 22 | variable "docker_ver" { 23 | default = "17.06.2.ol" 24 | } 25 | 26 | variable "oracle_linux_image_name" { 27 | default = "Oracle-Linux-7.5-2018.10.16-0" 28 | } 29 | 30 | variable "etcd_ver" { 31 | default = "v3.2.2" 32 | } 33 | 34 | variable "tenancy_ocid" {} 35 | variable "etcd_discovery_url" {} 36 | 37 | variable "count" { 38 | default = "1" 39 | } 40 | 41 | variable "control_plane_subnet_access" { 42 | description = "Whether instances in the control plane are launched in a public or private subnets" 43 | default = "public" 44 | } 45 | 46 | variable "etcd_docker_max_log_size" { 47 | description = "Maximum size of the etcd docker container json logs" 48 | default = "50m" 49 | } 50 | variable "etcd_docker_max_log_files" { 51 | description = "Maximum number of etcd docker container json logs to rotate" 52 | default = "5" 53 | } 54 | 55 | # iSCSI 56 | variable "etcd_iscsi_volume_create" { 57 | description = "Bool if an iscsi volume should be attached and mounted at the etcd volume mount point /etcd" 58 | default = false 59 | } 60 | 61 | variable "etcd_iscsi_volume_size" { 62 | description = "Size of iscsi volume to be created" 63 | default = 50 64 | } 65 | 66 | variable "assign_private_ip" { 67 | description = "Assign a static private ip based on CIDR block for that AD" 68 | default = false 69 | } 70 | -------------------------------------------------------------------------------- /instances/k8smaster/cloud_init/bootstrap.template.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | bootcmd: 4 | # Turn off SELinux 5 | - setenforce 0 6 | 7 | write_files: 8 | # setup script 9 | - path: "/root/setup.preflight.sh" 10 | permissions: "0777" 11 | encoding: "gzip+base64" 12 | content: | 13 | ${setup_preflight_sh_content} 14 | - path: "/root/setup.sh" 15 | permissions: "0777" 16 | encoding: "gzip+base64" 17 | content: | 18 | ${setup_template_sh_content} 19 | # Kube manifests 20 | - path: "/etc/kubernetes/manifests/kube-apiserver.yaml" 21 | permissions: "0755" 22 | encoding: "gzip+base64" 23 | content: | 24 | ${kube_apiserver_template_content} 25 | - path: "/etc/kubernetes/manifests/kube-controller-manager.yaml" 26 | permissions: "0755" 27 | encoding: "gzip+base64" 28 | content: | 29 | ${kube_controller_manager_template_content} 30 | - path: "/root/services/kube-dns.yaml" 31 | permissions: "0755" 32 | encoding: "gzip+base64" 33 | content: | 34 | ${kube_dns_template_content} 35 | - path: "/etc/kubernetes/manifests/kube-proxy.yaml" 36 | permissions: "0755" 37 | encoding: "gzip+base64" 38 | content: | 39 | ${kube_proxy_template_content} 40 | - path: "/etc/kubernetes/manifests/kube-scheduler.yaml" 41 | permissions: "0755" 42 | encoding: "gzip+base64" 43 | content: | 44 | ${kube_scheduler_template_content} 45 | - path: "/root/services/kubernetes-dashboard.yaml" 46 | permissions: "0755" 47 | encoding: "gzip+base64" 48 | content: | 49 | ${kube_dashboard_template_content} 50 | - path: "/etc/kubernetes/manifests/kube-rbac-role-binding.yaml" 51 | permissions: "0755" 52 | encoding: "gzip+base64" 53 | content: | 54 | ${kube_rbac_content} 55 | - path: "/root/cloud-controller-secret.yaml" 56 | permissions: "0600" 57 | encoding: "gzip+base64" 58 | content: | 59 | ${cloud_provider_secret_content} 60 | - path: "/root/flexvolume-driver-secret.yaml" 61 | permissions: "0600" 62 | encoding: "gzip+base64" 63 | content: | 64 | ${flexvolume_driver_secret_content} 65 | - path: "/root/volume-provisioner-secret.yaml" 66 | permissions: "0600" 67 | encoding: "gzip+base64" 68 | content: | 69 | ${volume_provisioner_secret_content} 70 | - path: "/etc/kubernetes/manifests/master-kubeconfig.yaml" 71 | permissions: "0755" 72 | encoding: "gzip+base64" 73 | content: | 74 | ${master_kubeconfig_template_content} 75 | # systemd services 76 | - path: "/root/services/kubelet.service" 77 | permissions: "0600" 78 | encoding: "gzip+base64" 79 | content: | 80 | ${kubelet_service_content} 81 | 82 | # Kube certs, tokens 83 | - path: "/etc/kubernetes/ssl/ca.pem" 84 | permissions: "0600" 85 | encoding: "gzip+base64" 86 | content: | 87 | ${ca-pem-content} 88 | - path: "/etc/kubernetes/ssl/ca-key.pem" 89 | permissions: "0600" 90 | encoding: "gzip+base64" 91 | content: | 92 | ${ca-key-content} 93 | - path: "/etc/kubernetes/ssl/apiserver.pem" 94 | permissions: "0600" 95 | encoding: "gzip+base64" 96 | content: | 97 | ${api-server-cert-content} 98 | - path: "/etc/kubernetes/ssl/apiserver-key.pem" 99 | permissions: "0600" 100 | encoding: "gzip+base64" 101 | content: | 102 | ${api-server-key-content} 103 | - path: "/etc/kubernetes/ssl/token_auth.csv" 104 | permissions: "0600" 105 | encoding: "gzip+base64" 106 | content: | 107 | ${api-token_auth_template_content} 108 | 109 | runcmd: 110 | - echo "Running k8s init..." 111 | - /root/setup.preflight.sh 112 | - echo "Finished k8s init." 113 | -------------------------------------------------------------------------------- /instances/k8smaster/datasources.tf: -------------------------------------------------------------------------------- 1 | # Prevent oci_core_images image list from changing underneath us. 2 | data "oci_core_images" "ImageOCID" { 3 | compartment_id = "${var.compartment_ocid}" 4 | display_name = "${var.oracle_linux_image_name}" 5 | } 6 | 7 | # Cloud call to get a list of Availability Domains 8 | data "oci_identity_availability_domains" "ADs" { 9 | compartment_id = "${var.tenancy_ocid}" 10 | } 11 | 12 | data "template_file" "setup-template" { 13 | template = "${file("${path.module}/scripts/setup.template.sh")}" 14 | 15 | vars = { 16 | domain_name = "${var.domain_name}" 17 | docker_ver = "${var.docker_ver}" 18 | etcd_ver = "${var.etcd_ver}" 19 | flannel_ver = "${var.flannel_ver}" 20 | flannel_network_cidr = "${var.flannel_network_cidr}" 21 | flannel_backend = "${var.flannel_backend}" 22 | k8s_ver = "${var.k8s_ver}" 23 | docker_max_log_size = "${var.master_docker_max_log_size}" 24 | docker_max_log_files = "${var.master_docker_max_log_files}" 25 | etcd_discovery_url = "${file("${path.root}/generated/discovery${var.etcd_discovery_url}")}" 26 | etcd_endpoints = "${var.etcd_endpoints}" 27 | cloud_controller_version = "${var.cloud_controller_version}" 28 | flexvolume_driver_version = "${var.flexvolume_driver_version}" 29 | volume_provisioner_version = "${var.volume_provisioner_version}" 30 | kubernetes_network_plugin = "${var.kubernetes_network_plugin}" 31 | } 32 | } 33 | 34 | data "template_file" "setup-preflight" { 35 | template = "${file("${path.module}/scripts/setup.preflight.sh")}" 36 | 37 | vars = { 38 | k8s_ver = "${var.k8s_ver}" 39 | } 40 | } 41 | 42 | data "template_file" "kube-apiserver" { 43 | template = "${file("${path.module}/manifests/kube-apiserver.yaml")}" 44 | 45 | vars = { 46 | api_server_count = "${var.api_server_count}" 47 | domain_name = "${var.domain_name}" 48 | k8s_ver = "${var.k8s_ver}" 49 | etcd_endpoints = "${var.etcd_endpoints}" 50 | } 51 | } 52 | 53 | data "template_file" "kubelet-service" { 54 | template = "${file("${path.module}/scripts/kubelet.service")}" 55 | 56 | vars = { 57 | k8s_ver = "${var.k8s_ver}" 58 | } 59 | } 60 | 61 | data "template_file" "kube-controller-manager" { 62 | template = "${file("${path.module}/manifests/kube-controller-manager.yaml")}" 63 | 64 | vars = { 65 | k8s_ver = "${var.k8s_ver}" 66 | flannel_network_cidr = "${var.flannel_network_cidr}" 67 | } 68 | } 69 | 70 | data "template_file" "kube-dns" { 71 | template = "${file("${path.module}/manifests/kube-dns.yaml")}" 72 | 73 | vars = { 74 | pillar_dns_domain = "cluster.local" 75 | k8s_dns_ver = "${var.k8s_dns_ver}" 76 | } 77 | } 78 | 79 | data "template_file" "kube-proxy" { 80 | template = "${file("${path.module}/manifests/kube-proxy.yaml")}" 81 | 82 | vars = { 83 | k8s_ver = "${var.k8s_ver}" 84 | flannel_network_cidr = "${var.flannel_network_cidr}" 85 | } 86 | } 87 | 88 | data "template_file" "kube-scheduler" { 89 | template = "${file("${path.module}/manifests/kube-scheduler.yaml")}" 90 | 91 | vars = { 92 | k8s_ver = "${var.k8s_ver}" 93 | } 94 | } 95 | 96 | data "template_file" "kube-dashboard" { 97 | template = "${file("${path.module}/manifests/kubernetes-dashboard.yaml")}" 98 | 99 | vars = { 100 | k8s_dashboard_ver = "${var.k8s_dashboard_ver}" 101 | } 102 | } 103 | 104 | data "template_file" "kube-rbac" { 105 | template = "${file("${path.module}/manifests/kube-rbac-role-binding.yaml")}" 106 | } 107 | 108 | data "template_file" "master-kubeconfig" { 109 | template = "${file("${path.module}/manifests/master-kubeconfig.template.yaml")}" 110 | } 111 | 112 | data "template_file" "token_auth_file" { 113 | template = "${file("${path.module}/scripts/token_auth.csv")}" 114 | 115 | vars { 116 | token_admin = "${var.k8s_apiserver_token_admin}" 117 | } 118 | } 119 | 120 | data "template_file" "kube_master_cloud_init_file" { 121 | template = "${file("${path.module}/cloud_init/bootstrap.template.yaml")}" 122 | 123 | vars = { 124 | k8s_ver = "${var.k8s_ver}" 125 | setup_preflight_sh_content = "${base64gzip(data.template_file.setup-preflight.rendered)}" 126 | setup_template_sh_content = "${base64gzip(data.template_file.setup-template.rendered)}" 127 | kube_apiserver_template_content = "${base64gzip(data.template_file.kube-apiserver.rendered)}" 128 | kube_controller_manager_template_content = "${base64gzip(data.template_file.kube-controller-manager.rendered)}" 129 | kube_dns_template_content = "${base64gzip(data.template_file.kube-dns.rendered)}" 130 | kube_proxy_template_content = "${base64gzip(data.template_file.kube-proxy.rendered)}" 131 | kube_dashboard_template_content = "${base64gzip(data.template_file.kube-dashboard.rendered)}" 132 | kube_rbac_content = "${base64gzip(data.template_file.kube-rbac.rendered)}" 133 | master_kubeconfig_template_content = "${base64gzip(data.template_file.master-kubeconfig.rendered)}" 134 | kube_scheduler_template_content = "${base64gzip(data.template_file.kube-scheduler.rendered)}" 135 | kubelet_service_content = "${base64gzip(data.template_file.kubelet-service.rendered)}" 136 | ca-pem-content = "${base64gzip(var.root_ca_pem)}" 137 | ca-key-content = "${base64gzip(var.root_ca_key)}" 138 | api-server-key-content = "${base64gzip(var.api_server_private_key_pem)}" 139 | api-server-cert-content = "${base64gzip(var.api_server_cert_pem)}" 140 | api-token_auth_template_content = "${base64gzip(data.template_file.token_auth_file.rendered)}" 141 | cloud_provider_secret_content = "${base64gzip(var.cloud_controller_secret)}" 142 | flexvolume_driver_secret_content = "${base64gzip(var.flexvolume_driver_secret)}" 143 | volume_provisioner_secret_content = "${base64gzip(var.volume_provisioner_secret)}" 144 | } 145 | } 146 | 147 | data "template_cloudinit_config" "master" { 148 | gzip = true 149 | base64_encode = true 150 | 151 | part { 152 | filename = "bootstrap.yaml" 153 | content_type = "text/cloud-config" 154 | content = "${data.template_file.kube_master_cloud_init_file.rendered}" 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /instances/k8smaster/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * The instances/k8smaster module provisions and configures one or more Kubernetes Master instances. 3 | */ 4 | 5 | resource "oci_core_instance" "TFInstanceK8sMaster" { 6 | count = "${var.count}" 7 | availability_domain = "${var.availability_domain}" 8 | compartment_id = "${var.compartment_ocid}" 9 | display_name = "${var.label_prefix}${var.display_name_prefix}-${count.index}" 10 | hostname_label = "${var.hostname_label_prefix}-${count.index}" 11 | image = "${lookup(data.oci_core_images.ImageOCID.images[0], "id")}" 12 | shape = "${var.shape}" 13 | 14 | create_vnic_details { 15 | subnet_id = "${var.subnet_id}" 16 | display_name = "${var.label_prefix}${var.display_name_prefix}-${count.index}" 17 | hostname_label = "${var.hostname_label_prefix}-${count.index}" 18 | assign_public_ip = "${(var.control_plane_subnet_access == "private") ? "false" : "true"}" 19 | private_ip = "${var.assign_private_ip == "true" ? cidrhost(lookup(var.network_cidrs,var.subnet_name), count.index+2) : ""}" 20 | } 21 | 22 | extended_metadata { 23 | roles = "masters" 24 | ssh_authorized_keys = "${var.ssh_public_key_openssh}" 25 | 26 | # Automate master instance configuration with cloud init run at launch time 27 | user_data = "${data.template_cloudinit_config.master.rendered}" 28 | tags = "group:k8s-master" 29 | } 30 | 31 | provisioner "remote-exec" { 32 | when = "destroy" 33 | 34 | inline = [ 35 | "nodeName=`getent hosts $(/usr/sbin/ip route get 1 | awk '{print $NF;exit}') | awk '{print $2}'`", 36 | "[ -e /usr/bin/kubectl ] && sudo kubectl --kubeconfig /etc/kubernetes/manifests/master-kubeconfig.yaml drain $nodeName --force", 37 | "[ -e /usr/bin/kubectl ] && sudo kubectl --kubeconfig /etc/kubernetes/manifests/master-kubeconfig.yaml delete node $nodeName", 38 | "exit 0", 39 | ] 40 | 41 | on_failure = "continue" 42 | 43 | connection { 44 | host = "${self.public_ip}" 45 | user = "opc" 46 | private_key = "${var.ssh_private_key}" 47 | agent = false 48 | timeout = "30s" 49 | } 50 | } 51 | 52 | timeouts { 53 | create = "60m" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kube-apiserver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-apiserver 5 | namespace: kube-system 6 | spec: 7 | hostNetwork: true 8 | containers: 9 | - name: kube-apiserver 10 | image: quay.io/coreos/hyperkube:v${k8s_ver}_coreos.0 11 | command: 12 | - /hyperkube 13 | - apiserver 14 | - --apiserver-count=${api_server_count} 15 | - --etcd-servers=${etcd_endpoints} 16 | - --allow-privileged=true 17 | - --service-cluster-ip-range=10.21.0.0/16 18 | - --secure-port=443 19 | - --insecure-port=8080 20 | - --insecure-bind-address=0.0.0.0 21 | - --audit-log-maxage=30 22 | - --audit-log-maxsize=100 23 | - --audit-log-path=/var/log/apiserver/audit.log 24 | - --feature-gates=AdvancedAuditing=false 25 | - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,Initializers 26 | - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem 27 | - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem 28 | - --client-ca-file=/etc/kubernetes/ssl/ca.pem 29 | - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem 30 | - --runtime-config=batch/v2alpha1,extensions/v1beta1=true,extensions/v1beta1/networkpolicies=true,admissionregistration.k8s.io/v1alpha1=true 31 | - --anonymous-auth=false 32 | - --authorization-mode=RBAC 33 | - --etcd-quorum-read=true 34 | - --kubelet-client-certificate=/etc/kubernetes/ssl/apiserver.pem 35 | - --kubelet-client-key=/etc/kubernetes/ssl/apiserver-key.pem 36 | - --token-auth-file=/etc/kubernetes/ssl/token_auth.csv 37 | ports: 38 | - containerPort: 443 39 | hostPort: 443 40 | name: https 41 | - containerPort: 8080 42 | hostPort: 8080 43 | name: local 44 | volumeMounts: 45 | - mountPath: /etc/kubernetes/auth 46 | name: auth-kubernetes 47 | readOnly: true 48 | - mountPath: /etc/kubernetes/ssl 49 | name: ssl-certs-kubernetes 50 | readOnly: true 51 | - mountPath: /etc/ssl/certs 52 | name: ssl-certs-host 53 | readOnly: true 54 | - mountPath: /var/log/apiserver 55 | name: apiserver-logs 56 | readOnly: false 57 | volumes: 58 | - hostPath: 59 | path: /etc/kubernetes/auth 60 | name: auth-kubernetes 61 | - hostPath: 62 | path: /etc/kubernetes/ssl 63 | name: ssl-certs-kubernetes 64 | - hostPath: 65 | path: /etc/ssl/certs 66 | name: ssl-certs-host 67 | - hostPath: 68 | path: /var/log/apiserver 69 | name: apiserver-logs 70 | -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kube-controller-manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-controller-manager 5 | namespace: kube-system 6 | spec: 7 | hostNetwork: true 8 | containers: 9 | - name: kube-controller-manager 10 | image: quay.io/coreos/hyperkube:v${k8s_ver}_coreos.0 11 | command: 12 | - /hyperkube 13 | - controller-manager 14 | - --master=http://127.0.0.1:8080 15 | - --cluster-cidr=${flannel_network_cidr} 16 | - --cluster-name=dev-cluster 17 | - --leader-elect=true 18 | - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem 19 | - --root-ca-file=/etc/kubernetes/ssl/ca.pem 20 | - --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem 21 | - --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem 22 | - --v=2 23 | - --allocate-node-cidrs=true 24 | livenessProbe: 25 | httpGet: 26 | host: 127.0.0.1 27 | path: /healthz 28 | port: 10252 29 | initialDelaySeconds: 15 30 | timeoutSeconds: 1 31 | volumeMounts: 32 | - mountPath: /etc/kubernetes/ssl 33 | name: ssl-certs-kubernetes 34 | readOnly: true 35 | - mountPath: /etc/kubernetes/ca 36 | name: ssl-certs-kubernetes 37 | readOnly: true 38 | - mountPath: /etc/ssl 39 | name: ssl-certs-host 40 | readOnly: true 41 | - mountPath: /etc/pki 42 | name: pki-certs-host 43 | readOnly: true 44 | - mountPath: /usr/libexec/kubernetes/kubelet-plugins 45 | name: plugins 46 | volumes: 47 | - hostPath: 48 | path: /etc/kubernetes/ssl 49 | name: ssl-certs-kubernetes 50 | - hostPath: 51 | path: /etc/ssl 52 | name: ssl-certs-host 53 | - hostPath: 54 | path: /etc/pki 55 | name: pki-certs-host 56 | - hostPath: 57 | path: /usr/libexec/kubernetes/kubelet-plugins 58 | name: plugins 59 | -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kube-dns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: kube-dns 5 | namespace: kube-system 6 | annotations: 7 | scheduler.alpha.kubernetes.io/critical-pod: '' 8 | scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' 9 | labels: 10 | addonmanager.kubernetes.io/mode: EnsureExists 11 | --- 12 | apiVersion: v1 13 | kind: ServiceAccount 14 | metadata: 15 | name: kube-dns 16 | namespace: kube-system 17 | labels: 18 | kubernetes.io/cluster-service: "true" 19 | addonmanager.kubernetes.io/mode: Reconcile 20 | --- 21 | apiVersion: v1 22 | kind: Service 23 | metadata: 24 | name: kube-dns 25 | namespace: kube-system 26 | labels: 27 | k8s-app: kube-dns 28 | kubernetes.io/cluster-service: "true" 29 | addonmanager.kubernetes.io/mode: Reconcile 30 | kubernetes.io/name: "KubeDNS" 31 | spec: 32 | clusterIP: 10.21.21.21 33 | selector: 34 | k8s-app: kube-dns 35 | ports: 36 | - name: dns 37 | port: 53 38 | protocol: UDP 39 | - name: dns-tcp 40 | port: 53 41 | protocol: TCP 42 | --- 43 | apiVersion: extensions/v1beta1 44 | kind: Deployment 45 | metadata: 46 | name: kube-dns 47 | namespace: kube-system 48 | labels: 49 | k8s-app: kube-dns 50 | kubernetes.io/cluster-service: "true" 51 | addonmanager.kubernetes.io/mode: Reconcile 52 | spec: 53 | strategy: 54 | rollingUpdate: 55 | maxSurge: 10% 56 | maxUnavailable: 0 57 | selector: 58 | matchLabels: 59 | k8s-app: kube-dns 60 | template: 61 | metadata: 62 | labels: 63 | k8s-app: kube-dns 64 | annotations: 65 | scheduler.alpha.kubernetes.io/critical-pod: '' 66 | spec: 67 | tolerations: 68 | - key: "CriticalAddonsOnly" 69 | operator: "Exists" 70 | volumes: 71 | - name: kube-dns-config 72 | configMap: 73 | name: kube-dns 74 | optional: true 75 | containers: 76 | - name: kubedns 77 | image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:${k8s_dns_ver} 78 | resources: 79 | limits: 80 | memory: 170Mi 81 | requests: 82 | cpu: 100m 83 | memory: 70Mi 84 | livenessProbe: 85 | httpGet: 86 | path: /healthcheck/kubedns 87 | port: 10054 88 | scheme: HTTP 89 | initialDelaySeconds: 60 90 | timeoutSeconds: 5 91 | successThreshold: 1 92 | failureThreshold: 5 93 | readinessProbe: 94 | httpGet: 95 | path: /readiness 96 | port: 8081 97 | scheme: HTTP 98 | initialDelaySeconds: 3 99 | timeoutSeconds: 5 100 | args: 101 | - --domain=${pillar_dns_domain}. 102 | - --dns-port=10053 103 | - --config-dir=/kube-dns-config 104 | - --v=2 105 | env: 106 | - name: PROMETHEUS_PORT 107 | value: "10055" 108 | ports: 109 | - containerPort: 10053 110 | name: dns-local 111 | protocol: UDP 112 | - containerPort: 10053 113 | name: dns-tcp-local 114 | protocol: TCP 115 | - containerPort: 10055 116 | name: metrics 117 | protocol: TCP 118 | volumeMounts: 119 | - name: kube-dns-config 120 | mountPath: /kube-dns-config 121 | - name: dnsmasq 122 | image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:${k8s_dns_ver} 123 | livenessProbe: 124 | httpGet: 125 | path: /healthcheck/dnsmasq 126 | port: 10054 127 | scheme: HTTP 128 | initialDelaySeconds: 60 129 | timeoutSeconds: 5 130 | successThreshold: 1 131 | failureThreshold: 5 132 | args: 133 | - -v=2 134 | - -logtostderr 135 | - -configDir=/etc/k8s/dns/dnsmasq-nanny 136 | - -restartDnsmasq=true 137 | - -- 138 | - -k 139 | - --cache-size=1000 140 | - --log-facility=- 141 | - --server=/${pillar_dns_domain}/127.0.0.1#10053 142 | - --server=/in-addr.arpa/127.0.0.1#10053 143 | - --server=/ip6.arpa/127.0.0.1#10053 144 | ports: 145 | - containerPort: 53 146 | name: dns 147 | protocol: UDP 148 | - containerPort: 53 149 | name: dns-tcp 150 | protocol: TCP 151 | resources: 152 | requests: 153 | cpu: 150m 154 | memory: 20Mi 155 | volumeMounts: 156 | - name: kube-dns-config 157 | mountPath: /etc/k8s/dns/dnsmasq-nanny 158 | - name: sidecar 159 | image: gcr.io/google_containers/k8s-dns-sidecar-amd64:${k8s_dns_ver} 160 | livenessProbe: 161 | httpGet: 162 | path: /metrics 163 | port: 10054 164 | scheme: HTTP 165 | initialDelaySeconds: 60 166 | timeoutSeconds: 5 167 | successThreshold: 1 168 | failureThreshold: 5 169 | args: 170 | - --v=2 171 | - --logtostderr 172 | - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${pillar_dns_domain},5,A 173 | - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${pillar_dns_domain},5,A 174 | ports: 175 | - containerPort: 10054 176 | name: metrics 177 | protocol: TCP 178 | resources: 179 | requests: 180 | memory: 20Mi 181 | cpu: 10m 182 | dnsPolicy: Default 183 | serviceAccountName: kube-dns 184 | -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kube-proxy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-proxy 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-proxy 8 | annotations: 9 | scheduler.alpha.kubernetes.io/critical-pod: '' 10 | scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' 11 | spec: 12 | hostNetwork: true 13 | containers: 14 | - name: kube-proxy 15 | image: quay.io/coreos/hyperkube:v${k8s_ver}_coreos.0 16 | command: 17 | - /hyperkube 18 | - proxy 19 | - --master=http://127.0.0.1:8080 20 | - --proxy-mode=iptables 21 | - --cluster-cidr=${flannel_network_cidr} 22 | securityContext: 23 | privileged: true 24 | volumeMounts: 25 | - mountPath: /etc/ssl/certs 26 | name: ssl-certs-host 27 | readOnly: true 28 | volumes: 29 | - hostPath: 30 | path: /usr/share/ca-certificates 31 | name: ssl-certs-host 32 | -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kube-rbac-role-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: system:default-sa 5 | subjects: 6 | - kind: ServiceAccount 7 | name: default 8 | namespace: kube-system 9 | roleRef: 10 | kind: ClusterRole 11 | name: cluster-admin 12 | apiGroup: rbac.authorization.k8s.io -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kube-scheduler.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-scheduler 5 | namespace: kube-system 6 | spec: 7 | hostNetwork: true 8 | containers: 9 | - name: kube-scheduler 10 | image: quay.io/coreos/hyperkube:v${k8s_ver}_coreos.0 11 | command: 12 | - /hyperkube 13 | - scheduler 14 | - --master=http://127.0.0.1:8080 15 | - --leader-elect=true 16 | livenessProbe: 17 | httpGet: 18 | host: 127.0.0.1 19 | path: /healthz 20 | port: 10251 21 | initialDelaySeconds: 15 22 | timeoutSeconds: 1 23 | -------------------------------------------------------------------------------- /instances/k8smaster/manifests/kubernetes-dashboard.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | labels: 5 | k8s-app: kubernetes-dashboard 6 | name: kubernetes-dashboard 7 | namespace: kube-system 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1beta1 10 | kind: ClusterRoleBinding 11 | metadata: 12 | name: kubernetes-dashboard 13 | labels: 14 | k8s-app: kubernetes-dashboard 15 | roleRef: 16 | apiGroup: rbac.authorization.k8s.io 17 | kind: ClusterRole 18 | name: cluster-admin 19 | subjects: 20 | - kind: ServiceAccount 21 | name: kubernetes-dashboard 22 | namespace: kube-system 23 | --- 24 | kind: Deployment 25 | apiVersion: extensions/v1beta1 26 | metadata: 27 | labels: 28 | k8s-app: kubernetes-dashboard 29 | name: kubernetes-dashboard 30 | namespace: kube-system 31 | spec: 32 | replicas: 1 33 | revisionHistoryLimit: 10 34 | selector: 35 | matchLabels: 36 | k8s-app: kubernetes-dashboard 37 | template: 38 | metadata: 39 | labels: 40 | k8s-app: kubernetes-dashboard 41 | spec: 42 | containers: 43 | - name: kubernetes-dashboard 44 | image: gcr.io/google_containers/kubernetes-dashboard-amd64:v${k8s_dashboard_ver} 45 | ports: 46 | - containerPort: 9090 47 | protocol: TCP 48 | args: 49 | livenessProbe: 50 | httpGet: 51 | path: / 52 | port: 9090 53 | initialDelaySeconds: 30 54 | timeoutSeconds: 30 55 | serviceAccountName: kubernetes-dashboard 56 | tolerations: 57 | - key: node-role.kubernetes.io/master 58 | effect: NoSchedule 59 | --- 60 | kind: Service 61 | apiVersion: v1 62 | metadata: 63 | labels: 64 | k8s-app: kubernetes-dashboard 65 | name: kubernetes-dashboard 66 | namespace: kube-system 67 | spec: 68 | ports: 69 | - port: 80 70 | targetPort: 9090 71 | selector: 72 | k8s-app: kubernetes-dashboard -------------------------------------------------------------------------------- /instances/k8smaster/manifests/master-kubeconfig.template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | clusters: 4 | - name: local 5 | cluster: 6 | server: http://localhost:8080 7 | contexts: 8 | - context: 9 | cluster: local 10 | name: kubelet-context 11 | current-context: kubelet-context 12 | -------------------------------------------------------------------------------- /instances/k8smaster/outputs.tf: -------------------------------------------------------------------------------- 1 | output "ids" { 2 | value = ["${oci_core_instance.TFInstanceK8sMaster.*.id}"] 3 | } 4 | 5 | output "private_ips" { 6 | value = ["${oci_core_instance.TFInstanceK8sMaster.*.private_ip}"] 7 | } 8 | 9 | output "public_ips" { 10 | value = ["${oci_core_instance.TFInstanceK8sMaster.*.public_ip}"] 11 | } 12 | -------------------------------------------------------------------------------- /instances/k8smaster/scripts/kubelet.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Requires=docker.service 3 | After=docker.service 4 | [Service] 5 | EnvironmentFile=/etc/environment_params 6 | ExecStart=/usr/bin/kubelet \ 7 | --allow-privileged=true \ 8 | --cluster_dns=10.21.21.21 \ 9 | --cluster_domain=cluster.local \ 10 | --container-runtime=docker \ 11 | --hostname-override=__FQDN_HOSTNAME__ \ 12 | --kubeconfig=/etc/kubernetes/manifests/master-kubeconfig.yaml \ 13 | --require-kubeconfig=true \ 14 | --network-plugin=cni \ 15 | --node-labels node-role.kubernetes.io/master= \ 16 | --pod-manifest-path=/etc/kubernetes/manifests \ 17 | --register-node=true \ 18 | --cloud-provider=external \ 19 | --feature-gates=ExperimentalCriticalPodAnnotation=true,Accelerators=true \ 20 | --provider-id=__NODE_ID_PREFIX__.__NODE_ID_SUFFIX__ \ 21 | --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ 22 | __SWAP_OPTION__ --v=2 23 | Restart=always 24 | RestartSec=10 25 | [Install] 26 | WantedBy=multi-user.target 27 | -------------------------------------------------------------------------------- /instances/k8smaster/scripts/setup.preflight.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | EXTERNAL_IP=$(curl -s -m 10 http://whatismyip.akamai.com/) 4 | 5 | mkdir -p /etc/kubernetes/auth /etc/kubernetes/manifests/ 6 | 7 | bash -x /root/setup.sh | tee -a /root/setup.log 8 | -------------------------------------------------------------------------------- /instances/k8smaster/scripts/setup.template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | EXTERNAL_IP=$(curl -s -m 10 http://whatismyip.akamai.com/) 4 | NAMESPACE=$(echo -n "${domain_name}" | sed "s/\.oraclevcn\.com//g") 5 | FQDN_HOSTNAME=$(hostname -f) 6 | 7 | # Pull instance metadata 8 | curl -sL --retry 3 http://169.254.169.254/opc/v1/instance/ | tee /tmp/instance_meta.json 9 | 10 | ETCD_ENDPOINTS=${etcd_endpoints} 11 | export HOSTNAME=$(hostname) 12 | 13 | export IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip addr show | grep -Po 'inet \K[\d.]+') 14 | 15 | SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2) 16 | 17 | ## k8s_ver swap option 18 | ###################################### 19 | k8sversion="${k8s_ver}" 20 | 21 | if [[ $k8sversion =~ ^[0-1]+\.[0-7]+ ]]; then 22 | SWAP_OPTION="" 23 | else 24 | SWAP_OPTION="--fail-swap-on=false" 25 | fi 26 | 27 | ## etcd 28 | ###################################### 29 | 30 | ## Disable TX checksum offloading so we don't break VXLAN 31 | ###################################### 32 | BROADCOM_DRIVER=$(lsmod | grep bnxt_en | awk '{print $1}') 33 | if [[ -n "$${BROADCOM_DRIVER}" ]]; then 34 | echo "Disabling hardware TX checksum offloading" 35 | ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off 36 | fi 37 | 38 | # Download etcdctl client 39 | curl -L --retry 3 https://github.com/coreos/etcd/releases/download/${etcd_ver}/etcd-${etcd_ver}-linux-amd64.tar.gz -o /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz 40 | tar zxf /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz -C /tmp/ && cp /tmp/etcd-${etcd_ver}-linux-amd64/etcd* /usr/local/bin/ 41 | 42 | # Wait for etcd to become active (through the LB) 43 | until [ $(/usr/local/bin/etcdctl --endpoints ${etcd_endpoints} cluster-health | grep '^cluster ' | grep -c 'is healthy$') == "1" ]; do 44 | echo "Waiting for etcd cluster to be healthy" 45 | sleep 1 46 | done 47 | 48 | ## Docker 49 | ###################################### 50 | until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done 51 | 52 | cat < /etc/sysconfig/docker 53 | OPTIONS="--selinux-enabled --log-opt max-size=${docker_max_log_size} --log-opt max-file=${docker_max_log_files}" 54 | DOCKER_CERT_PATH=/etc/docker 55 | GOTRACEBACK=crash 56 | EOF 57 | 58 | systemctl daemon-reload 59 | systemctl enable docker 60 | systemctl start docker 61 | 62 | # Output /etc/environment_params 63 | echo "IPV4_PRIVATE_0=$IP_LOCAL" >>/etc/environment_params 64 | echo "ETCD_IP=$ETCD_ENDPOINTS" >>/etc/environment_params 65 | echo "FQDN_HOSTNAME=$FQDN_HOSTNAME" >>/etc/environment_params 66 | 67 | # Drop firewall rules 68 | iptables -F 69 | 70 | cat < /etc/yum.repos.d/kubernetes.repo 71 | [kubernetes] 72 | name=Kubernetes 73 | baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 74 | enabled=1 75 | gpgcheck=1 76 | repo_gpgcheck=1 77 | gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg 78 | https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg 79 | EOF 80 | 81 | # Disable SELinux and firewall 82 | sudo sed -i s/SELINUX=enforcing/SELINUX=permissive/ /etc/selinux/config 83 | setenforce 0 84 | systemctl stop firewalld.service 85 | systemctl disable firewalld.service 86 | 87 | ## Install Flex Volume Driver for OCI 88 | ##################################### 89 | mkdir -p /usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/ 90 | curl -L --retry 3 https://github.com/oracle/oci-flexvolume-driver/releases/download/${flexvolume_driver_version}/oci -o/usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/oci 91 | chmod a+x /usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/oci 92 | mv /root/flexvolume-driver-secret.yaml /usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/config.yaml 93 | 94 | ## Install kubelet, kubectl, and kubernetes-cni 95 | ############################################### 96 | yum-config-manager --add-repo http://yum.kubernetes.io/repos/kubernetes-el7-x86_64 97 | yum search -y kubernetes 98 | 99 | VER_IN_REPO=$(repoquery --nvr --show-duplicates kubelet | sort --version-sort | grep ${k8s_ver} | tail -n 1) 100 | if [[ -z "$${VER_IN_REPO}" ]]; then 101 | MAJOR_VER=$(echo ${k8s_ver} | cut -d. -f-2) 102 | echo "Falling back to latest version available in: $MAJOR_VER" 103 | VER_IN_REPO=$(repoquery --nvr --show-duplicates kubelet | sort --version-sort | grep $MAJOR_VER | tail -n 1) 104 | echo "Installing kubelet version: $VER_IN_REPO" 105 | yum install -y $VER_IN_REPO 106 | ## Replace kubelet binary since rpm at the exact k8s_ver was not available. 107 | curl -L --retry 3 http://storage.googleapis.com/kubernetes-release/release/v${k8s_ver}/bin/linux/amd64/kubelet -o /bin/kubelet && chmod 755 /bin/kubelet 108 | else 109 | echo "Installing kubelet version: $VER_IN_REPO" 110 | yum install -y $VER_IN_REPO 111 | fi 112 | 113 | # Check if kubernetes-cni was automatically installed as a dependency 114 | K8S_CNI=$(rpm -qa | grep kubernetes-cni) 115 | if [[ -z "$${K8S_CNI}" ]]; then 116 | echo "Installing: $K8S_CNI" 117 | yum install -y kubernetes-cni 118 | else 119 | echo "$K8S_CNI already installed" 120 | fi 121 | 122 | curl -L --retry 3 http://storage.googleapis.com/kubernetes-release/release/v${k8s_ver}/bin/linux/amd64/kubectl -o /bin/kubectl && chmod 755 /bin/kubectl 123 | 124 | # Pull etcd docker image from registry 125 | docker pull quay.io/coreos/etcd:${etcd_ver} 126 | 127 | # Start etcd proxy container 128 | docker run -d \ 129 | -p 2380:2380 -p 2379:2379 \ 130 | -v /etc/ssl/certs/ca-bundle.crt:/etc/ssl/certs/ca-bundle.crt \ 131 | --net=host \ 132 | --restart=always \ 133 | quay.io/coreos/etcd:${etcd_ver} \ 134 | /usr/local/bin/etcd \ 135 | -discovery ${etcd_discovery_url} \ 136 | --proxy on 137 | 138 | ## kubelet for the master 139 | systemctl daemon-reload 140 | read NODE_ID_0 NODE_ID_1 <<< $(jq -r '.id' /tmp/instance_meta.json | perl -pe 's/(.*?\.){4}\K/ /g' | perl -pe 's/\.+\s/ /g') 141 | sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" \ 142 | -e "s/__SWAP_OPTION__/$SWAP_OPTION/g" \ 143 | -e "s/__NODE_ID_PREFIX__/$NODE_ID_0/g" \ 144 | -e "s/__NODE_ID_SUFFIX__/$NODE_ID_1/g" \ 145 | /root/services/kubelet.service >/etc/systemd/system/kubelet.service 146 | systemctl daemon-reload 147 | systemctl enable kubelet 148 | systemctl start kubelet 149 | 150 | until kubectl get all; do sleep 1 && echo -n "."; done 151 | 152 | ## Wait for k8s master to be available. There is a possible race on pod networks otherwise. 153 | until [ "$(curl localhost:8080/healthz 2>/dev/null)" == "ok" ]; do 154 | sleep 3 155 | done 156 | 157 | 158 | case "${kubernetes_network_plugin}" in 159 | flannel) 160 | # Install flannel 161 | kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/${flannel_ver}/Documentation/k8s-manifests/kube-flannel-rbac.yml 162 | 163 | ## This could be done better 164 | curl -sSL https://raw.githubusercontent.com/coreos/flannel/${flannel_ver}/Documentation/kube-flannel.yml | \ 165 | sed -e "s#10.244.0.0/16#${flannel_network_cidr}#g" \ 166 | -e "s#vxlan#${flannel_backend}#g" | \ 167 | kubectl apply -f - 168 | ;; 169 | canal) 170 | # Setup Kube CNI - canal 171 | # https://docs.projectcalico.org/v3.0/getting-started/kubernetes/installation/hosted/canal/ 172 | kubectl apply -f https://docs.projectcalico.org/v3.0/getting-started/kubernetes/installation/hosted/canal/rbac.yaml 173 | curl -sSL kubectl apply -f https://docs.projectcalico.org/v3.0/getting-started/kubernetes/installation/hosted/canal/canal.yaml | \ 174 | sed -e "s#10.244.0.0/16#${flannel_network_cidr}#g" \ 175 | -e "s#vxlan#${flannel_backend}#g" | \ 176 | kubectl apply -f - 177 | ;; 178 | esac 179 | 180 | # Install oci cloud controller manager 181 | kubectl apply -f /root/cloud-controller-secret.yaml 182 | kubectl apply -f https://github.com/oracle/oci-cloud-controller-manager/releases/download/${cloud_controller_version}/oci-cloud-controller-manager-rbac.yaml 183 | curl -sSL https://github.com/oracle/oci-cloud-controller-manager/releases/download/${cloud_controller_version}/oci-cloud-controller-manager.yaml | \ 184 | sed -e "s#10.244.0.0/16#${flannel_network_cidr}#g" | \ 185 | kubectl apply -f - 186 | 187 | ## install kube-dns 188 | kubectl create -f /root/services/kube-dns.yaml 189 | 190 | ## install kubernetes-dashboard 191 | kubectl create -f /root/services/kubernetes-dashboard.yaml 192 | 193 | ## Install Volume Provisioner of OCI 194 | kubectl create secret generic oci-volume-provisioner -n kube-system --from-file=config.yaml=/root/volume-provisioner-secret.yaml 195 | kubectl apply -f https://github.com/oracle/oci-volume-provisioner/releases/download/${volume_provisioner_version}/oci-volume-provisioner-rbac.yaml 196 | kubectl apply -f https://github.com/oracle/oci-volume-provisioner/releases/download/${volume_provisioner_version}/oci-volume-provisioner.yaml 197 | kubectl apply -f https://github.com/oracle/oci-volume-provisioner/releases/download/${volume_provisioner_version}/storage-class.yaml 198 | kubectl apply -f https://github.com/oracle/oci-volume-provisioner/releases/download/${volume_provisioner_version}/storage-class-ext3.yaml 199 | 200 | ## Mark OCI StorageClass as the default 201 | kubectl patch storageclass oci -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' 202 | 203 | rm -f /root/volume-provisioner-secret.yaml 204 | 205 | yum install -y nfs-utils 206 | 207 | echo "Finished running setup.sh" 208 | -------------------------------------------------------------------------------- /instances/k8smaster/scripts/token_auth.csv: -------------------------------------------------------------------------------- 1 | ${token_admin},admin,admin,"system:masters" -------------------------------------------------------------------------------- /instances/k8smaster/variables.tf: -------------------------------------------------------------------------------- 1 | # BMCS 2 | variable "availability_domain" {} 3 | 4 | variable "compartment_ocid" {} 5 | variable "display_name_prefix" {} 6 | variable "hostname_label_prefix" {} 7 | 8 | variable "flannel_network_cidr" {} 9 | variable "flannel_backend" {} 10 | 11 | variable "kubernetes_network_plugin" {} 12 | 13 | variable "count" { 14 | default = "1" 15 | } 16 | 17 | variable "control_plane_subnet_access" { 18 | description = "Whether instances in the control plane are launched in a public or private subnets" 19 | default = "public" 20 | } 21 | 22 | variable "network_cidrs" { 23 | type = "map" 24 | } 25 | variable "subnet_id" {} 26 | variable "subnet_name" {} 27 | variable "domain_name" {} 28 | variable "shape" {} 29 | variable "tenancy_ocid" {} 30 | 31 | variable "label_prefix" { 32 | default = "" 33 | } 34 | 35 | # Instance 36 | variable "ssh_public_key_openssh" {} 37 | 38 | variable "docker_ver" { 39 | default = "17.06.2.ol" 40 | } 41 | 42 | variable "oracle_linux_image_name" { 43 | default = "Oracle-Linux-7.5-2018.10.16-0" 44 | } 45 | 46 | variable "etcd_ver" { 47 | default = "v3.2.2" 48 | } 49 | 50 | variable "flannel_ver" { 51 | default = "v0.9.1" 52 | } 53 | 54 | variable "ssh_private_key" {} 55 | 56 | # Kubernetes 57 | variable "k8s_ver" { 58 | default = "1.8.5" 59 | } 60 | 61 | variable "k8s_dashboard_ver" { 62 | default = "1.6.3" 63 | } 64 | 65 | variable "k8s_dns_ver" { 66 | default = "1.14.2" 67 | } 68 | 69 | variable "api_server_count" {} 70 | 71 | variable "root_ca_pem" {} 72 | variable "root_ca_key" {} 73 | variable "api_server_private_key_pem" {} 74 | variable "api_server_cert_pem" {} 75 | variable "k8s_apiserver_token_admin" {} 76 | 77 | # etcd 78 | variable "etcd_discovery_url" {} 79 | variable "etcd_endpoints" {} 80 | 81 | variable "master_docker_max_log_size" { 82 | description = "Maximum size of the k8s master docker container json logs" 83 | default = "50m" 84 | } 85 | variable "master_docker_max_log_files" { 86 | description = "Maximum number of k8s master docker container json logs to rotate" 87 | default = "5" 88 | } 89 | 90 | variable "cloud_controller_version" {} 91 | variable "cloud_controller_secret" {} 92 | 93 | variable "flexvolume_driver_version" {} 94 | variable "flexvolume_driver_secret" {} 95 | 96 | variable "volume_provisioner_version" {} 97 | variable "volume_provisioner_secret" {} 98 | 99 | variable "assign_private_ip" { 100 | description = "Assign a static private ip based on CIDR block for that AD" 101 | default = false 102 | } 103 | -------------------------------------------------------------------------------- /instances/k8sworker/cloud_init/bootstrap.template.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | bootcmd: 4 | # Turn off SELinux 5 | - setenforce 0 6 | 7 | write_files: 8 | # setup script 9 | - path: "/root/setup.preflight.sh" 10 | permissions: "0777" 11 | encoding: "gzip+base64" 12 | content: | 13 | ${setup_preflight_sh_content} 14 | - path: "/root/setup.sh" 15 | permissions: "0777" 16 | encoding: "gzip+base64" 17 | content: | 18 | ${setup_template_sh_content} 19 | # Kube manifests 20 | - path: "/etc/kubernetes/manifests/kube-proxy.yaml" 21 | permissions: "0755" 22 | encoding: "gzip+base64" 23 | content: | 24 | ${kube_proxy_template_content} 25 | - path: "/etc/kubernetes/manifests/worker-kubeconfig.yaml" 26 | permissions: "0755" 27 | encoding: "gzip+base64" 28 | content: | 29 | ${worker_kubeconfig_template_content} 30 | 31 | # systemd services 32 | 33 | - path: "/root/services/kubelet.service" 34 | permissions: "0600" 35 | encoding: "gzip+base64" 36 | content: | 37 | ${kubelet_service_content} 38 | 39 | # Kube certs 40 | - path: "/etc/kubernetes/ssl/ca.pem" 41 | permissions: "0600" 42 | encoding: "gzip+base64" 43 | content: | 44 | ${ca-pem-content} 45 | - path: "/etc/kubernetes/ssl/ca-key.pem" 46 | permissions: "0600" 47 | encoding: "gzip+base64" 48 | content: | 49 | ${ca-key-content} 50 | - path: "/etc/kubernetes/ssl/apiserver.pem" 51 | permissions: "0600" 52 | encoding: "gzip+base64" 53 | content: | 54 | ${api-server-cert-content} 55 | - path: "/etc/kubernetes/ssl/apiserver-key.pem" 56 | permissions: "0600" 57 | encoding: "gzip+base64" 58 | content: | 59 | ${api-server-key-content} 60 | 61 | ${reverse_proxy-content} 62 | 63 | runcmd: 64 | - echo "Running k8s init..." 65 | - /root/setup.preflight.sh 66 | - echo "Finished k8s init." 67 | -------------------------------------------------------------------------------- /instances/k8sworker/datasources.tf: -------------------------------------------------------------------------------- 1 | # Prevent oci_core_images image list from changing underneath us. 2 | data "oci_core_images" "ImageOCID" { 3 | compartment_id = "${var.compartment_ocid}" 4 | display_name = "${var.oracle_linux_image_name}" 5 | } 6 | 7 | # Cloud call to get a list of Availability Domains 8 | data "oci_identity_availability_domains" "ADs" { 9 | compartment_id = "${var.tenancy_ocid}" 10 | } 11 | 12 | data "template_file" "setup-template" { 13 | template = "${file("${path.module}/scripts/setup.template.sh")}" 14 | 15 | vars = { 16 | master_lb = "${var.master_lb}" 17 | domain_name = "${var.domain_name}" 18 | docker_ver = "${var.docker_ver}" 19 | k8s_ver = "${var.k8s_ver}" 20 | docker_max_log_size = "${var.worker_docker_max_log_size}" 21 | docker_max_log_files = "${var.worker_docker_max_log_files}" 22 | worker_iscsi_volume_mount = "${var.worker_iscsi_volume_mount}" 23 | flexvolume_driver_version = "${var.flexvolume_driver_version}" 24 | reverse_proxy_setup = "${var.reverse_proxy_setup}" 25 | } 26 | } 27 | 28 | data "template_file" "setup-preflight" { 29 | template = "${file("${path.module}/scripts/setup.preflight.sh")}" 30 | 31 | vars = { 32 | k8s_ver = "${var.k8s_ver}" 33 | } 34 | } 35 | 36 | data "template_file" "kube-proxy" { 37 | template = "${file("${path.module}/manifests/kube-proxy.template.yaml")}" 38 | 39 | vars = { 40 | master_lb = "${var.master_lb}" 41 | k8s_ver = "${var.k8s_ver}" 42 | domain_name = "${var.domain_name}" 43 | flannel_network_cidr = "${var.flannel_network_cidr}" 44 | } 45 | } 46 | 47 | data "template_file" "worker-kubeconfig" { 48 | template = "${file("${path.module}/manifests/worker-kubeconfig.template.yaml")}" 49 | 50 | vars = { 51 | master_lb = "${var.master_lb}" 52 | k8s_ver = "${var.k8s_ver}" 53 | domain_name = "${var.domain_name}" 54 | } 55 | } 56 | 57 | data "template_file" "kubelet-service" { 58 | template = "${file("${path.module}/scripts/kubelet.service")}" 59 | 60 | vars = { 61 | master_lb = "${var.master_lb}" 62 | k8s_ver = "${var.k8s_ver}" 63 | domain_name = "${var.domain_name}" 64 | region = "${var.region}" 65 | zone = "${element(split(":",var.availability_domain),1)}" 66 | } 67 | } 68 | 69 | data "template_file" "kube_worker_cloud_init_file" { 70 | template = "${file("${path.module}/cloud_init/bootstrap.template.yaml")}" 71 | 72 | vars = { 73 | k8s_ver = "${var.k8s_ver}" 74 | setup_preflight_sh_content = "${base64gzip(data.template_file.setup-preflight.rendered)}" 75 | setup_template_sh_content = "${base64gzip(data.template_file.setup-template.rendered)}" 76 | kube_proxy_template_content = "${base64gzip(data.template_file.kube-proxy.rendered)}" 77 | worker_kubeconfig_template_content = "${base64gzip(data.template_file.worker-kubeconfig.rendered)}" 78 | kubelet_service_content = "${base64gzip(data.template_file.kubelet-service.rendered)}" 79 | ca-pem-content = "${base64gzip(var.root_ca_pem)}" 80 | ca-key-content = "${base64gzip(var.root_ca_key)}" 81 | api-server-key-content = "${base64gzip(var.api_server_private_key_pem)}" 82 | api-server-cert-content = "${base64gzip(var.api_server_cert_pem)}" 83 | reverse_proxy-content = "${var.reverse_proxy_clount_init}" 84 | } 85 | } 86 | 87 | data "template_cloudinit_config" "master" { 88 | gzip = true 89 | base64_encode = true 90 | 91 | part { 92 | filename = "bootstrap.yaml" 93 | content_type = "text/cloud-config" 94 | content = "${data.template_file.kube_worker_cloud_init_file.rendered}" 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /instances/k8sworker/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * The instances/k8sworker module provisions and configures one or more Kubernetes Worker instances. 3 | */ 4 | 5 | resource "oci_core_instance" "TFInstanceK8sWorker" { 6 | count = "${var.count}" 7 | availability_domain = "${var.availability_domain}" 8 | compartment_id = "${var.compartment_ocid}" 9 | display_name = "${var.label_prefix}${var.display_name_prefix}-${count.index}" 10 | hostname_label = "${var.hostname_label_prefix}-${count.index}" 11 | image = "${lookup(data.oci_core_images.ImageOCID.images[0], "id")}" 12 | shape = "${var.shape}" 13 | subnet_id = "${var.subnet_id}" 14 | 15 | extended_metadata { 16 | roles = "nodes" 17 | ssh_authorized_keys = "${var.ssh_public_key_openssh}" 18 | 19 | # Automate worker instance configuration with cloud init run at launch time 20 | user_data = "${data.template_cloudinit_config.master.rendered}" 21 | tags = "group:k8s-worker" 22 | } 23 | 24 | # TODO handle scenario when control_plane_subnet_access = "private" 25 | provisioner "remote-exec" { 26 | when = "destroy" 27 | 28 | inline = [ 29 | "nodeName=`getent hosts $(/usr/sbin/ip route get 1 | awk '{print $NF;exit}') | awk '{print $2}'`", 30 | "[ -e /usr/bin/kubectl ] && sudo kubectl --kubeconfig /etc/kubernetes/manifests/worker-kubeconfig.yaml drain $nodeName --force", 31 | "[ -e /usr/bin/kubectl ] && sudo kubectl --kubeconfig /etc/kubernetes/manifests/worker-kubeconfig.yaml delete node $nodeName", 32 | "exit 0", 33 | ] 34 | 35 | on_failure = "continue" 36 | 37 | connection { 38 | host = "${self.public_ip}" 39 | user = "opc" 40 | private_key = "${var.ssh_private_key}" 41 | agent = false 42 | timeout = "30s" 43 | } 44 | } 45 | 46 | timeouts { 47 | create = "60m" 48 | } 49 | } 50 | 51 | resource "oci_core_volume" "TFVolumeK8sWorker" { 52 | count = "${var.worker_iscsi_volume_create ? var.count : 0}" 53 | availability_domain = "${var.availability_domain}" 54 | compartment_id = "${var.compartment_ocid}" 55 | display_name = "block-volume-${var.hostname_label_prefix}-${count.index}" 56 | size_in_gbs = "${var.worker_iscsi_volume_size}" 57 | } 58 | 59 | resource "oci_core_volume_attachment" "TFVolumeAttachmentK8sWorker" { 60 | count = "${var.worker_iscsi_volume_create ? var.count : 0}" 61 | attachment_type = "iscsi" 62 | compartment_id = "${var.compartment_ocid}" 63 | instance_id = "${oci_core_instance.TFInstanceK8sWorker.*.id[count.index]}" 64 | volume_id = "${oci_core_volume.TFVolumeK8sWorker.*.id[count.index]}" 65 | depends_on = ["oci_core_instance.TFInstanceK8sWorker", "oci_core_volume.TFVolumeK8sWorker"] 66 | } 67 | -------------------------------------------------------------------------------- /instances/k8sworker/manifests/kube-proxy.template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-proxy 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-proxy 8 | annotations: 9 | scheduler.alpha.kubernetes.io/critical-pod: '' 10 | scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' 11 | spec: 12 | hostNetwork: true 13 | containers: 14 | - name: kube-proxy 15 | image: quay.io/coreos/hyperkube:v${k8s_ver}_coreos.0 16 | command: 17 | - /hyperkube 18 | - proxy 19 | - --master=${master_lb} 20 | - --kubeconfig=/etc/kubernetes/manifests/worker-kubeconfig.yaml 21 | - --proxy-mode=iptables 22 | - --cluster-cidr=${flannel_network_cidr} 23 | - --masquerade-all 24 | - --hostname-override=__FQDN_HOSTNAME__ 25 | - --v=2 26 | securityContext: 27 | privileged: true 28 | volumeMounts: 29 | - mountPath: /etc/ssl/certs 30 | name: "ssl-certs" 31 | - mountPath: /etc/kubernetes/manifests/worker-kubeconfig.yaml 32 | name: "kubeconfig" 33 | readOnly: true 34 | - mountPath: /etc/kubernetes/ssl 35 | name: "etc-kube-ssl" 36 | readOnly: true 37 | volumes: 38 | - name: "ssl-certs" 39 | hostPath: 40 | path: "/usr/share/ca-certificates" 41 | - name: "kubeconfig" 42 | hostPath: 43 | path: "/etc/kubernetes/manifests/worker-kubeconfig.yaml" 44 | - name: "etc-kube-ssl" 45 | hostPath: 46 | path: "/etc/kubernetes/ssl" 47 | -------------------------------------------------------------------------------- /instances/k8sworker/manifests/worker-kubeconfig.template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | clusters: 4 | - name: local 5 | cluster: 6 | certificate-authority: /etc/kubernetes/ssl/ca.pem 7 | server: ${master_lb} 8 | users: 9 | - name: kubelet 10 | user: 11 | client-certificate: /etc/kubernetes/ssl/apiserver.pem 12 | client-key: /etc/kubernetes/ssl/apiserver-key.pem 13 | contexts: 14 | - context: 15 | cluster: local 16 | user: kubelet 17 | name: kubelet-context 18 | current-context: kubelet-context 19 | -------------------------------------------------------------------------------- /instances/k8sworker/output.tf: -------------------------------------------------------------------------------- 1 | output "ids" { 2 | value = ["${oci_core_instance.TFInstanceK8sWorker.*.id}"] 3 | } 4 | 5 | output "private_ips" { 6 | value = ["${oci_core_instance.TFInstanceK8sWorker.*.private_ip}"] 7 | } 8 | 9 | output "public_ips" { 10 | value = ["${oci_core_instance.TFInstanceK8sWorker.*.public_ip}"] 11 | } 12 | 13 | output "instance_host_names" { 14 | value = ["${oci_core_instance.TFInstanceK8sWorker.*.display_name}"] 15 | } 16 | -------------------------------------------------------------------------------- /instances/k8sworker/scripts/kubelet.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Requires=docker.service 3 | After=docker.service 4 | [Service] 5 | EnvironmentFile=/etc/environment_params 6 | ExecStart=/usr/bin/kubelet \ 7 | --allow-privileged=true \ 8 | --cluster_dns=10.21.21.21 \ 9 | --cluster_domain=cluster.local \ 10 | --container-runtime=docker \ 11 | --docker=unix:///var/run/docker.sock \ 12 | --hostname-override=__FQDN_HOSTNAME__ \ 13 | --kubeconfig=/etc/kubernetes/manifests/worker-kubeconfig.yaml \ 14 | --require-kubeconfig=true \ 15 | --network-plugin=cni \ 16 | --node-labels node-role.kubernetes.io/node=,failure-domain.beta.kubernetes.io/region=${region},failure-domain.beta.kubernetes.io/zone=${zone},node.info/external.ipaddress=__EXT_IP__,node.info/availability.domain=__AVAILABILITY_DOMAIN__,node.info/compartment.id_prefix=__COMPARTMENT_ID_PREFIX__,node.info/compartment.id_suffix=__COMPARTMENT_ID_SUFFIX__,node.info/node.id_prefix=__NODE_ID_PREFIX__,node.info/node.id_suffix=__NODE_ID_SUFFIX__,node.info/node.shape=__NODE_SHAPE__ \ 17 | --pod-manifest-path=/etc/kubernetes/manifests \ 18 | --register-node=true \ 19 | --serialize-image-pulls=false \ 20 | --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem \ 21 | --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \ 22 | --feature-gates=ExperimentalCriticalPodAnnotation=true,Accelerators=true \ 23 | --eviction-hard=memory.available<500Mi,nodefs.available<2Gi,imagefs.available<2Gi \ 24 | --cloud-provider=external \ 25 | --provider-id=__NODE_ID_PREFIX__.__NODE_ID_SUFFIX__ \ 26 | __SWAP_OPTION__ --v=2 27 | Restart=always 28 | RestartSec=10 29 | [Install] 30 | WantedBy=multi-user.target 31 | -------------------------------------------------------------------------------- /instances/k8sworker/scripts/setup.preflight.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | EXTERNAL_IP=$(curl -s -m 10 http://whatismyip.akamai.com/) 4 | 5 | mkdir -p /etc/kubernetes/manifests /etc/kubernetes/auth 6 | 7 | # add tools 8 | curl --retry 3 http://stedolan.github.io/jq/download/linux64/jq -o /usr/local/bin/jq && chmod +x /usr/local/bin/jq 9 | bash -x /root/setup.sh 2>&1 | tee -a /root/setup.log 10 | -------------------------------------------------------------------------------- /instances/k8sworker/scripts/setup.template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | EXTERNAL_IP=$(curl -s -m 10 http://whatismyip.akamai.com/) 4 | NAMESPACE=$(echo -n "${domain_name}" | sed "s/\.oraclevcn\.com//g") 5 | FQDN_HOSTNAME=$(hostname -f) 6 | 7 | # Pull instance metadata 8 | curl -sL --retry 3 http://169.254.169.254/opc/v1/instance/ | tee /tmp/instance_meta.json 9 | 10 | ## Create policy file that blocks autostart of services on install 11 | printf '#!/bin/sh\necho "All runlevel operations denied by policy" >&2\nexit 101\n' >/tmp/policy-rc.d && chmod +x /tmp/policy-rc.d 12 | export K8S_API_SERVER_LB=${master_lb} 13 | export RANDFILE=$(mktemp) 14 | export HOSTNAME=$(hostname) 15 | 16 | export IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip addr show | grep -Po 'inet \K[\d.]+') 17 | 18 | SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2) 19 | export WORKER_IP=$IP_LOCAL 20 | 21 | ## k8s_ver swap option 22 | ###################################### 23 | k8sversion="${k8s_ver}" 24 | 25 | if [[ $k8sversion =~ ^[0-1]+\.[0-7]+ ]]; then 26 | SWAP_OPTION="" 27 | else 28 | SWAP_OPTION="--fail-swap-on=false" 29 | fi 30 | 31 | ## Disable TX checksum offloading so we don't break VXLAN 32 | ###################################### 33 | BROADCOM_DRIVER=$(lsmod | grep bnxt_en | awk '{print $1}') 34 | if [[ -n "$${BROADCOM_DRIVER}" ]]; then 35 | echo "Disabling hardware TX checksum offloading" 36 | ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off 37 | fi 38 | 39 | ## Setup NVMe drives and mount at /var/lib/docker 40 | ###################################### 41 | NVMEVGNAME="NVMeVG" 42 | NVMELVNAME="DockerVol" 43 | NVMEDEVS=$(lsblk -I259 -pn -oNAME -d) 44 | if [[ ! -z "$${NVMEDEVS}" ]]; then 45 | lvs $${NVMEVGNAME}/$${NVMELVNAME} --noheadings --logonly 1>/dev/null 46 | if [ $$? -ne 0 ]; then 47 | pvcreate $${NVMEDEVS} 48 | vgcreate $${NVMEVGNAME} $${NVMEDEVS} 49 | lvcreate --extents 100%FREE --name $${NVMELVNAME} $${NVMEVGNAME} $${NVMEDEVS} 50 | mkfs -t xfs /dev/$${NVMEVGNAME}/$${NVMELVNAME} 51 | mkdir -p /var/lib/docker 52 | mount -t xfs /dev/$${NVMEVGNAME}/$${NVMELVNAME} /var/lib/docker 53 | echo "/dev/$${NVMEVGNAME}/$${NVMELVNAME} /var/lib/docker xfs rw,relatime,seclabel,attr2,inode64,noquota 0 2" >> /etc/fstab 54 | fi 55 | fi 56 | 57 | ## Login iSCSI volume mount and create filesystem 58 | ###################################### 59 | iqn=$(iscsiadm --mode discoverydb --type sendtargets --portal 169.254.2.2:3260 --discover| cut -f2 -d" ") 60 | 61 | if [ -n "$${iqn}" ]; then 62 | echo "iSCSI Login $${iqn}" 63 | iscsiadm -m node -o new -T $${iqn} -p 169.254.2.2:3260 64 | iscsiadm -m node -o update -T $${iqn} -n node.startup -v automatic 65 | iscsiadm -m node -T $${iqn} -p 169.254.2.2:3260 -l 66 | # Wait for device to apear... 67 | until [[ -e "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" ]]; do sleep 1 && echo -n "."; done 68 | # If the volume has been created and formatted before but it's just a new instance this may fail 69 | # but if so ignore and carry on. 70 | mkfs -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1"; 71 | echo "$$(readlink -f /dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1) ${worker_iscsi_volume_mount} xfs defaults,noatime,_netdev 0 2" >> /etc/fstab 72 | mkdir -p ${worker_iscsi_volume_mount} 73 | mount -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" ${worker_iscsi_volume_mount} 74 | fi 75 | 76 | until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done 77 | 78 | cat < /etc/sysconfig/docker 79 | OPTIONS="--selinux-enabled --log-opt max-size=${docker_max_log_size} --log-opt max-file=${docker_max_log_files}" 80 | DOCKER_CERT_PATH=/etc/docker 81 | GOTRACEBACK=crash 82 | EOF 83 | 84 | systemctl daemon-reload 85 | systemctl enable docker 86 | systemctl start docker 87 | 88 | ## Output /etc/environment_params 89 | ###################################### 90 | echo "IPV4_PRIVATE_0=$IP_LOCAL" >>/etc/environment_params 91 | echo "ETCD_IP=$ETCD_ENDPOINTS" >>/etc/environment_params 92 | echo "K8S_API_SERVER_LB=$K8S_API_SERVER_LB" >>/etc/environment_params 93 | echo "FQDN_HOSTNAME=$FQDN_HOSTNAME" >>/etc/environment_params 94 | 95 | ## Drop firewall rules 96 | ###################################### 97 | iptables -F 98 | 99 | cat < /etc/yum.repos.d/kubernetes.repo 100 | [kubernetes] 101 | name=Kubernetes 102 | baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 103 | enabled=1 104 | gpgcheck=1 105 | repo_gpgcheck=1 106 | gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg 107 | https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg 108 | EOF 109 | 110 | # Disable SELinux and firewall 111 | setenforce 0 112 | sudo sed -i s/SELINUX=enforcing/SELINUX=permissive/ /etc/selinux/config 113 | systemctl stop firewalld.service 114 | systemctl disable firewalld.service 115 | 116 | ## Install Flex Volume Driver for OCI 117 | ##################################### 118 | mkdir -p /usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/ 119 | curl -L --retry 3 https://github.com/oracle/oci-flexvolume-driver/releases/download/${flexvolume_driver_version}/oci -o/usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/oci 120 | chmod a+x /usr/libexec/kubernetes/kubelet-plugins/volume/exec/oracle~oci/oci 121 | 122 | 123 | ## Install kubelet, kubectl, and kubernetes-cni 124 | ############################################### 125 | yum-config-manager --add-repo http://yum.kubernetes.io/repos/kubernetes-el7-x86_64 126 | yum search -y kubernetes 127 | 128 | VER_IN_REPO=$(repoquery --nvr --show-duplicates kubelet | sort --version-sort | grep ${k8s_ver} | tail -n 1) 129 | if [[ -z "$${VER_IN_REPO}" ]]; then 130 | MAJOR_VER=$(echo ${k8s_ver} | cut -d. -f-2) 131 | echo "Falling back to latest version available in: $MAJOR_VER" 132 | VER_IN_REPO=$(repoquery --nvr --show-duplicates kubelet | sort --version-sort | grep $MAJOR_VER | tail -n 1) 133 | echo "Installing kubelet version: $VER_IN_REPO" 134 | yum install -y $VER_IN_REPO 135 | ## Replace kubelet binary since rpm at the exact k8s_ver was not available. 136 | curl -L --retry 3 http://storage.googleapis.com/kubernetes-release/release/v${k8s_ver}/bin/linux/amd64/kubelet -o /bin/kubelet && chmod 755 /bin/kubelet 137 | else 138 | echo "Installing kubelet version: $VER_IN_REPO" 139 | yum install -y $VER_IN_REPO 140 | fi 141 | 142 | # Check if kubernetes-cni was automatically installed as a dependency 143 | K8S_CNI=$(rpm -qa | grep kubernetes-cni) 144 | if [[ -z "$${K8S_CNI}" ]]; then 145 | echo "Installing: $K8S_CNI" 146 | yum install -y kubernetes-cni 147 | else 148 | echo "$K8S_CNI already installed" 149 | fi 150 | 151 | curl -L --retry 3 http://storage.googleapis.com/kubernetes-release/release/v${k8s_ver}/bin/linux/amd64/kubectl -o /bin/kubectl && chmod 755 /bin/kubectl 152 | 153 | ## FQDN constructed from live environment since DNS label for the subnet is optional 154 | sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" /etc/kubernetes/manifests/kube-proxy.yaml >/tmp/kube-proxy.yaml 155 | cat /tmp/kube-proxy.yaml >/etc/kubernetes/manifests/kube-proxy.yaml 156 | 157 | ## kubelet for the worker 158 | ###################################### 159 | systemctl daemon-reload 160 | 161 | AVAILABILITY_DOMAIN=$(jq -r '.availabilityDomain' /tmp/instance_meta.json | sed 's/:/-/g') 162 | read COMPARTMENT_ID_0 COMPARTMENT_ID_1 <<< $(jq -r '.compartmentId' /tmp/instance_meta.json | perl -pe 's/(.*?\.){4}\K/ /g' | perl -pe 's/\.+\s/ /g') 163 | read NODE_ID_0 NODE_ID_1 <<< $(jq -r '.id' /tmp/instance_meta.json | perl -pe 's/(.*?\.){4}\K/ /g' | perl -pe 's/\.+\s/ /g') 164 | NODE_SHAPE=$(jq -r '.shape' /tmp/instance_meta.json) 165 | 166 | sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" \ 167 | -e "s/__EXT_IP__/$EXTERNAL_IP/g" \ 168 | -e "s/__AVAILABILITY_DOMAIN__/$AVAILABILITY_DOMAIN/g" \ 169 | -e "s/__COMPARTMENT_ID_PREFIX__/$COMPARTMENT_ID_0/g" \ 170 | -e "s/__COMPARTMENT_ID_SUFFIX__/$COMPARTMENT_ID_1/g" \ 171 | -e "s/__NODE_ID_PREFIX__/$NODE_ID_0/g" \ 172 | -e "s/__NODE_ID_SUFFIX__/$NODE_ID_1/g" \ 173 | -e "s/__NODE_SHAPE__/$NODE_SHAPE/g" \ 174 | -e "s/__SWAP_OPTION__/$SWAP_OPTION/g" \ 175 | /root/services/kubelet.service > /etc/systemd/system/kubelet.service 176 | 177 | ${reverse_proxy_setup} 178 | ## Wait for k8s master to be available. There is a possible race on pod networks otherwise. 179 | until [ "$(curl -k --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem $K8S_API_SERVER_LB/healthz 2>/dev/null)" == "ok" ]; do 180 | sleep 3 181 | done 182 | 183 | # Setup CUDA devices before starting kubelet, so it detects the gpu(s) 184 | /sbin/modprobe nvidia 185 | if [ "$?" -eq 0 ]; then 186 | # Create the /dev/nvidia* files by running nvidia-smi 187 | nvidia-smi 188 | fi 189 | 190 | /sbin/modprobe nvidia-uvm 191 | if [ "$?" -eq 0 ]; then 192 | # Find out the major device number used by the nvidia-uvm driver 193 | DEVICE=$(grep nvidia-uvm /proc/devices | awk '{print $1}') 194 | mknod -m 666 /dev/nvidia-uvm c $DEVICE 0 195 | fi 196 | 197 | sleep $[ ( $RANDOM % 10 ) + 1 ]s 198 | systemctl daemon-reload 199 | systemctl enable kubelet 200 | systemctl start kubelet 201 | 202 | yum install -y nfs-utils 203 | 204 | ###################################### 205 | echo "Finished running setup.sh" 206 | -------------------------------------------------------------------------------- /instances/k8sworker/variables.tf: -------------------------------------------------------------------------------- 1 | # BMCS 2 | variable "availability_domain" {} 3 | 4 | variable "compartment_ocid" {} 5 | variable "display_name_prefix" {} 6 | variable "hostname_label_prefix" {} 7 | variable "flannel_network_cidr" {} 8 | 9 | variable "count" { 10 | default = "1" 11 | } 12 | 13 | variable "subnet_id" {} 14 | variable "domain_name" {} 15 | variable "region" {} 16 | variable "shape" {} 17 | variable "tenancy_ocid" {} 18 | 19 | variable "label_prefix" { 20 | default = "" 21 | } 22 | 23 | # Instance 24 | variable "ssh_public_key_openssh" {} 25 | 26 | variable "ssh_private_key" {} 27 | 28 | variable "docker_ver" { 29 | default = "17.06.2.ol" 30 | } 31 | 32 | variable "oracle_linux_image_name" { 33 | default = "Oracle-Linux-7.5-2018.10.16-0" 34 | } 35 | 36 | # Kubernetes 37 | variable "master_lb" {} 38 | 39 | variable "k8s_ver" { 40 | default = "1.8.5" 41 | } 42 | 43 | variable "root_ca_pem" {} 44 | variable "root_ca_key" {} 45 | variable "api_server_private_key_pem" {} 46 | variable "api_server_cert_pem" {} 47 | 48 | variable "worker_docker_max_log_size" { 49 | description = "Maximum size of the k8s worker docker container json logs" 50 | default = "50m" 51 | } 52 | variable "worker_docker_max_log_files" { 53 | description = "Maximum number of the k8s worker docker container json logs to rotate" 54 | default = "5" 55 | } 56 | 57 | # iSCSI 58 | variable "worker_iscsi_volume_create" { 59 | description = "Bool if an iscsi volume should be attached and mounted at /var/lib/docker" 60 | default = false 61 | } 62 | 63 | variable "worker_iscsi_volume_size" { 64 | description = "Size of iscsi volume to be created" 65 | default = 50 66 | } 67 | 68 | variable "worker_iscsi_volume_mount" { 69 | description = "Mount point of iscsi volume" 70 | default = "/var/lib/docker" 71 | } 72 | 73 | variable "flexvolume_driver_version" {} 74 | 75 | variable "reverse_proxy_setup" {} 76 | 77 | variable "reverse_proxy_clount_init" {} 78 | -------------------------------------------------------------------------------- /kubernetes/kubeconfig/kubeconfig.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "kubeconfig" { 2 | template = </dev/null 14 | # kubectl delete service nginx &>/dev/null 15 | } 16 | 17 | function log_msg() { 18 | local logger=$(basename "$0") 19 | printf "[${logger}] $1\n" 20 | } 21 | 22 | function check_tf_output() { 23 | local output_name="$1" 24 | if [[ -z $(terraform output $1) ]]; then 25 | echo terraform output variable $1 could not be resolved 26 | echo This script needs to be able to run: \"terraform output $1\" 27 | exit 1 28 | fi 29 | } 30 | 31 | # SSH to a node ($1), run command ($2)). 32 | function ssh_run_command() { 33 | local node="$1" 34 | local command="$2" 35 | ssh -i /tmp/instances_id_rsa -oBatchMode=yes -oConnectTimeout=10 -o StrictHostKeyChecking=no \ 36 | -o UserKnownHostsFile=/dev/null -q opc@${node} ${command} 2>&1 | tr -d '\r\n' 37 | } 38 | 39 | function check_ssh_connectivity() { 40 | log_msg " Checking SSH connectivity to each node (from this host)..." 41 | check_tf_output "master_public_ips" 42 | for master in $(terraform output master_public_ips | sed "s/,/ /g"); do 43 | ret=$(ssh_run_command "${master}" "echo connected") 44 | if [[ $ret != "connected" ]]; then 45 | log_msg " [FAILED] Could not ssh to master ${master}" 46 | log_msg " Only IPs in $(terraform output master_ssh_ingress_cidr) are allowed to SSH to this node" 47 | log_msg " If it is not correct, set master_ssh_ingress in terraform.tfvars to a CIDR that includes the IP \ 48 | of this host and run terraform plan and apply" 49 | exit 1 50 | fi 51 | done 52 | 53 | check_tf_output "worker_public_ips" 54 | for worker in $(terraform output worker_public_ips | sed "s/,/ /g"); do 55 | ret=$(ssh_run_command "${worker}" "echo connected") 56 | if [[ $ret != "connected" ]]; then 57 | log_msg " [FAILED] Could not ssh to worker ${worker}" 58 | log_msg " Only IPs in $(terraform output worker_ssh_ingress_cidr) are allowed to SSH to this node" 59 | log_msg " If it is not correct, set worker_ssh_ingress in terraform.tfvars to a CIDR that includes the IP \ 60 | of this host and run terraform plan and apply" 61 | exit 1 62 | fi 63 | done 64 | 65 | check_tf_output "etcd_public_ips" 66 | for etcd in $(terraform output etcd_public_ips | sed "s/,/ /g"); do 67 | ret=$(ssh_run_command "${etcd}" "echo connected") 68 | if [[ $ret != "connected" ]]; then 69 | log_msg " [FAILED] Could not ssh to etcd instance ${etcd}" 70 | log_msg " Only IPs in $(terraform output etcd_ssh_ingress_cidr) are allowed to SSH to this node" 71 | log_msg " If it is not correct, set etcd_ssh_ingress in terraform.tfvars to a CIDR that includes the IP \ 72 | of this host and run terraform plan and apply" 73 | exit 1 74 | fi 75 | done 76 | } 77 | 78 | function check_cloud_init_finished() { 79 | log_msg " Checking whether instance bootstrap has completed on each node..." 80 | for master in $(terraform output master_public_ips | sed "s/,/ /g"); do 81 | ret=$(ssh_run_command "${master}" "sudo test -e /var/lib/cloud/instance/boot-finished && echo true") 82 | if [[ $ret != "true" ]]; then 83 | log_msg " [FAILED] cloud-init has not finished running on master ${master}" 84 | log_msg " If this does not complete soon, log into the BMC instance and examine the /var/log/cloud-init-output.log file." 85 | exit 1 86 | fi 87 | ret=$(ssh_run_command "${master}" "sudo grep --only-matching -m 1 'Finished running setup.sh' /root/setup.log") 88 | if [[ $ret != "Finished running setup.sh" ]]; then 89 | log_msg " [FAILED] cloud-init has not run successfully on master ${master}" 90 | log_msg " Log into the BMC instance and examine the /root/setup.log file." 91 | exit 1 92 | fi 93 | done 94 | 95 | for worker in $(terraform output worker_public_ips | sed "s/,/ /g"); do 96 | ret=$(ssh_run_command "${worker}" "sudo test -e /var/lib/cloud/instance/boot-finished && echo true") 97 | if [[ $ret != "true" ]]; then 98 | log_msg " [FAILED] cloud-init has not finished running on worker ${worker}" 99 | log_msg " If this does not complete soon, log into the BMC instance and examine the /root/setup.log file." 100 | exit 1 101 | fi 102 | ret=$(ssh_run_command "${worker}" "sudo grep --only-matching -m 1 'Finished running setup.sh' /root/setup.log") 103 | if [[ $ret != "Finished running setup.sh" ]]; then 104 | log_msg " [FAILED] cloud-init has not run successfully on worker ${worker}" 105 | log_msg " Log into the BMC instance and examine the /root/setup.log file." 106 | exit 1 107 | fi 108 | done 109 | 110 | for etcd in $(terraform output etcd_public_ips | sed "s/,/ /g"); do 111 | ret=$(ssh_run_command "${etcd}" "sudo test -e /var/lib/cloud/instance/boot-finished && echo true") 112 | if [[ $ret != "true" ]]; then 113 | log_msg " [FAILED] cloud-init has not finished running on etcd node ${etcd}" 114 | log_msg " If this does not complete soon, log into the BMC instance and examine the /root/setup.log file." 115 | exit 1 116 | fi 117 | done 118 | } 119 | 120 | function check_system_services() { 121 | 122 | log_msg " Checking whether expected system services are running on each node..." 123 | for master in $(terraform output master_public_ips | sed "s/,/ /g"); do 124 | ret=$(ssh_run_command "${master}" "sudo systemctl status docker 2>&1 | grep --only-matching 'Active: active' | tr -d '\r\n'") 125 | if [[ $ret != "Active: active" ]]; then 126 | log_msg " [FAILED] expected docker service is not running on master $master" 127 | exit 1 128 | fi 129 | ret=$(ssh_run_command "${master}" "sudo systemctl status kubelet 2>&1 | grep --only-matching 'Active: active' | tr -d '\r\n'") 130 | if [[ $ret != "Active: active" ]]; then 131 | log_msg " [FAILED] expected kubelet service is not running on master $master" 132 | exit 1 133 | fi 134 | 135 | ret=$(ssh_run_command "${master}" "sudo docker ps | grep --only-matching \"hyperkube proxy\" | tr -d '\r\n'") 136 | if [[ $ret != "hyperkube proxy" ]]; then 137 | log_msg " [FAILED] expected hyperkube proxy service is not running on master $master" 138 | exit 1 139 | fi 140 | done 141 | 142 | for worker in $(terraform output worker_public_ips | sed "s/,/ /g"); do 143 | ret=$(ssh_run_command "${worker}" "sudo systemctl status docker 2>&1 | grep --only-matching 'Active: active' | tr -d '\r\n'") 144 | if [[ $ret != "Active: active" ]]; then 145 | log_msg " [FAILED] expected docker service is not running on worker $worker" 146 | exit 1 147 | fi 148 | ret=$(ssh_run_command "${worker}" "sudo systemctl status kubelet 2>&1 | grep --only-matching 'Active: active' | tr -d '\r\n'") 149 | if [[ $ret != "Active: active" ]]; then 150 | log_msg " [FAILED] expected kubelet service is not running on worker $worker" 151 | exit 1 152 | fi 153 | ret=$(ssh_run_command "${worker}" "sudo docker ps | grep --only-matching \"hyperkube proxy\" | tr -d '\r\n'") 154 | if [[ $ret != "hyperkube proxy" ]]; then 155 | log_msg " [FAILED] expected hyperkube proxy service is not running on worker $worker" 156 | exit 1 157 | fi 158 | 159 | done 160 | 161 | for etcd in $(terraform output etcd_public_ips | sed "s/,/ /g"); do 162 | ret=$(ssh_run_command "${etcd}" "sudo systemctl status docker 2>&1 | grep --only-matching 'Active: active' | tr -d '\r\n'") 163 | if [[ $ret != "Active: active" ]]; then 164 | log_msg " [FAILED] expected docker service is not running on etcd $etcd" 165 | exit 1 166 | fi 167 | done 168 | 169 | } 170 | 171 | function check_get_nodes() { 172 | log_msg " Running 'kubectl get nodes' a number of times through the master LB..." 173 | expected_nodes=$(expr $(terraform output worker_public_ips | tr -cd , | wc -c) + 1) 174 | for i in {1..20}; do 175 | count=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | wc -l) 176 | if [ "$count" -lt $expected_nodes ]; then 177 | log_msg " [FAILED] kubectl get nodes reported less healthy nodes than expected" 178 | kubectl get nodes 179 | exit 1 180 | fi 181 | done 182 | } 183 | 184 | function check_kube-dns() { 185 | log_msg " Checking status of kube-dns pod..." 186 | output=$(kubectl get pods --namespace=kube-system -l k8s-app=kube-dns 2>&1 | grep kube-dns | awk '{print $3}') 187 | if [[ $output != "Running" ]]; then 188 | log_msg " [FAILED] kube-dns is not running in the cluster" 189 | exit 1 190 | fi 191 | 192 | output=$(kubectl get pods --namespace=kube-system -l k8s-app=kube-dns 2>&1 | grep kube-dns | awk '{print $2}') 193 | if [[ $output != "3/3" ]]; then 194 | log_msg " [FAILED] expected 3/3 kube-dns pods to be running in the cluster" 195 | exit 1 196 | fi 197 | } 198 | 199 | function check_kube-flannel() { 200 | log_msg " Checking status of kube-flannel pod..." 201 | output=$(kubectl get pods --namespace=kube-system -l app=flannel 2>&1 | grep kube-flannel | awk '{print $3}' | grep -vi running) 202 | if [[ ! -z $output ]]; then 203 | log_msg " [FAILED] kube-flannel is not running in the cluster" 204 | exit 1 205 | fi 206 | } 207 | 208 | function check_nginx_deployment() { 209 | log_msg " Checking an app deployment of nginx exposed as a service..." 210 | expected_nodes=$(expr $(terraform output worker_public_ips | tr -cd , | wc -c) + 2) 211 | kubectl run nginx --image="nginx" --port=80 --replicas=${expected_nodes} 1>/dev/null 212 | kubectl expose deployment nginx --type NodePort 1>/dev/null 213 | 214 | nodePort="UNSET" 215 | max_tries=0 216 | until [ $nodePort != "UNSET" ] || [ $max_tries -eq 10 ]; do 217 | nodePort=$(kubectl get svc nginx --output=jsonpath='{range .spec.ports[0]}{.nodePort}') 218 | max_tries=$((max_tries + 1)) 219 | done 220 | 221 | if [[ $nodePort == "UNSET" ]]; then 222 | log_msg " [FAILED] could not retrieve the NodePort of the nginx service" 223 | exit 1 224 | fi 225 | 226 | # Avoid possible hang if port is not ready 227 | sleep 10 228 | for i in {1..5}; do 229 | for worker in $(terraform output worker_public_ips | sed "s/,/ /g"); do 230 | output=$(curl --max-time 30 http://$worker:$nodePort 2>/dev/null) 231 | if [[ -z "$output" ]]; then 232 | log_msg " [FAILED] nginx deployment service is not accessible on http://$worker:$nodePort" 233 | exit 1 234 | fi 235 | if [[ $output != *"Welcome to nginx!"* ]]; then 236 | log_msg " [FAILED] nginx deployment service is not accessible on http://$worker:$nodePort" 237 | exit 1 238 | fi 239 | done 240 | done 241 | } 242 | 243 | function print_success() { 244 | cat </tmp/instances_id_rsa 259 | chmod 600 /tmp/instances_id_rsa 260 | 261 | control_plane_subnet_access=$(terraform output control_plane_subnet_access) 262 | if [[ $control_plane_subnet_access == "private" ]]; then 263 | echo This script does not currently support checking private clusters 264 | exit 1 265 | fi 266 | 267 | log_msg "Running some basic checks on Kubernetes cluster...." 268 | 269 | check_ssh_connectivity 270 | check_cloud_init_finished 271 | check_system_services 272 | check_get_nodes 273 | check_kube-dns 274 | check_kube-flannel 275 | check_nginx_deployment 276 | print_success 277 | 278 | exit 0 279 | -------------------------------------------------------------------------------- /terraform.example.tfvars: -------------------------------------------------------------------------------- 1 | # OCI authentication 2 | 3 | #tenancy_ocid = "ocid1.tenancy.oc1..aaaaaaaa763cu5f3m7qpzwnvr2shs3o26ftrn7fkgz55cpzgxmglgtui3v7q" 4 | #compartment_ocid = "ocid1.compartment.oc1..aaaaaaaaidy3jl7bdmiwfryo6myhdnujcuug5zxzoclsz7vpfzw4bggng7iq" 5 | #fingerprint = "ed:51:83:3b:d2:04:f4:af:9d:7b:17:96:dd:8a:99:bc" 6 | #private_key_path = "/tmp/oci_api_key.pem" 7 | #user_ocid = "ocid1.user.oc1..aaaaaaaa5fy2l5aki6z2bzff5yrrmlahiif44vzodeetygxmpulq3mbnckya" 8 | 9 | # CCM user 10 | 11 | #cloud_controller_user_ocid = "ocid1.tenancy.oc1..aaaaaaaa763cu5f3m7qpzwnvr2shs3o26ftrn7fkgz55cpzgxmglgtui3v7q" 12 | #cloud_controller_user_fingerprint = "ed:51:83:3b:d2:04:f4:af:9d:7b:17:96:dd:8a:99:bc" 13 | #cloud_controller_user_private_key_path = "/tmp/oci_api_key.pem" 14 | 15 | #etcdShape = "VM.Standard1.2" 16 | #k8sMasterShape = "VM.Standard1.8" 17 | #k8sWorkerShape = "VM.Standard1.8" 18 | 19 | #etcdAd1Count = "1" 20 | #etcdAd2Count = "1" 21 | #etcdAd3Count = "1" 22 | 23 | #k8sMasterAd1Count = "1" 24 | #k8sMasterAd2Count = "1" 25 | #k8sMasterAd3Count = "1" 26 | 27 | #k8sWorkerAd1Count = "1" 28 | #k8sWorkerAd2Count = "1" 29 | #k8sWorkerAd3Count = "1" 30 | 31 | #etcdLBShape = "100Mbps" 32 | #k8sMasterLBShape = "100Mbps" 33 | 34 | #etcd_ssh_ingress = "10.0.0.0/16" 35 | #etcd_ssh_ingress = "0.0.0.0/0" 36 | #etcd_cluster_ingress = "10.0.0.0/16" 37 | #master_ssh_ingress = "0.0.0.0/0" 38 | #worker_ssh_ingress = "0.0.0.0/0" 39 | #master_https_ingress = "0.0.0.0/0" 40 | #worker_nodeport_ingress = "0.0.0.0/0" 41 | #worker_nodeport_ingress = "10.0.0.0/16" 42 | 43 | #control_plane_subnet_access = "public" 44 | #k8s_master_lb_access = "public" 45 | #natInstanceShape = "VM.Standard1.2" 46 | #nat_instance_ad1_enabled = "true" 47 | #nat_instance_ad2_enabled = "false" 48 | #nat_instance_ad3_enabled = "true" 49 | #nat_ssh_ingress = "0.0.0.0/0" 50 | #public_subnet_http_ingress = "0.0.0.0/0" 51 | #public_subnet_https_ingress = "0.0.0.0/0" 52 | 53 | #worker_iscsi_volume_create is a bool not a string 54 | #worker_iscsi_volume_create = true 55 | #worker_iscsi_volume_size = 100 56 | 57 | #etcd_iscsi_volume_create = true 58 | #etcd_iscsi_volume_size = 50 59 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Terraform Kubernetes Installer for OCI Tests 2 | 3 | ## About 4 | 5 | The Terraform Kubernetes Installer for OCI Tests provides a set of tests including: 6 | 7 | - Terraform static config validation 8 | - Cluster Creation Tests 9 | 10 | Tests can be run locally and are are run against **every** commit to the main branch by default. Successful test results are also required before merging PRs into the main branch, although this is not currently automatic. 11 | 12 | ## Running Tests Locally on the CLI (in your own tenancy) 13 | 14 | #### Prerequisites 15 | 16 | - Set up the general prerequisites as defined [here](../README.md#Prerequisites) 17 | - Install [Python](https://www.python.org/downloads) 2.7 or later 18 | - Install required Python packages (below) 19 | - Create a _terraform.tfvars_ file in the project root that specifies your the required keys and OCIDs for your tenancy, user, and compartment 20 | 21 | ```bash 22 | # Install required Python packages 23 | $ pip install -r requirements.txt 24 | ``` 25 | 26 | ```bash 27 | # start from the included example 28 | $ cp terraform.example.tfvars terraform.tfvars 29 | # specify private_key_path, fingerprint, tenancy_ocid, compartment_ocid, user_ocid, and region. 30 | ``` 31 | 32 | ```bash 33 | $ python2.7 ./create/runner.py 34 | ``` 35 | 36 | ## Running Tests Locally using the Wercker CLI (in your own tenancy) 37 | 38 | #### Prerequisites 39 | 40 | - Install [Docker](https://docs.docker.com/engine/installation/) 41 | - Provide Terraform the value of the required keys and OCIDs in the container through environment variables prefixed with `X_TF_VAR`: 42 | 43 | ```bash 44 | $ cat /tmp/bmcs_api_key.pem | pbcopy 45 | $ export X_TF_VAR_private_key=`pbpaste` 46 | $ export X_TF_VAR_fingerprint=... 47 | $ export X_TF_VAR_tenancy_ocid=ocid1.tenancy.oci... 48 | $ export X_TF_VAR_compartment_ocid=ocid1.compartment.oc1... 49 | $ export X_TF_VAR_user_ocid=ocid1.user.oc1... 50 | $ export X_TF_VAR_region=... 51 | $ cat /tmp/cloud_controller_bmcs_api_key.pem | pbcopy 52 | $ export X_TF_VAR_cloud_controller_user_ocid=ocid1.user.oc1... 53 | $ export X_TF_VAR_cloud_controller_user_fingerprint=... 54 | $ export X_TF_VAR_cloud_controller_user_private_key=`pbpaste` 55 | ``` 56 | 57 | ```bash 58 | $ wercker build 59 | $ wercker deploy 60 | ``` 61 | 62 | ### Notes 63 | 64 | - By default, the tests will create a series of clusters with Terraform, verify them, then destroy them 65 | - The tests use their own _cluster_ configuration (instance shapes, etc) defined in resources/*.tfvars 66 | 67 | -------------------------------------------------------------------------------- /tests/create/runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import argparse 4 | import json 5 | import os 6 | import select 7 | import subprocess 8 | import sys 9 | import time 10 | import requests 11 | from requests.packages.urllib3.exceptions import InsecureRequestWarning 12 | requests.packages.urllib3.disable_warnings(InsecureRequestWarning) 13 | import traceback 14 | 15 | TEST_ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/..") 16 | ROOT_DIR = os.path.abspath(TEST_ROOT_DIR + "/..") 17 | TEST_NAME = "createtests" 18 | 19 | def _banner(as_banner, bold): 20 | if as_banner: 21 | if bold: 22 | print "********************************************************" 23 | else: 24 | print "--------------------------------------------------------" 25 | 26 | 27 | def _test_log(string): 28 | # Extra precautionary measure 29 | if "ocid1.compartment" not in string and "ssh_authorized_keys" not in string: 30 | print string 31 | else: 32 | first = string.split(":") 33 | print first[0] + ":************************************************" 34 | 35 | 36 | def _log(string, as_banner=False, bold=False): 37 | _banner(as_banner, bold) 38 | print string 39 | _banner(as_banner, bold) 40 | 41 | 42 | def _process_stream(stream, read_fds, global_buf, line_buf): 43 | char = stream.read(1) 44 | if char == '': 45 | read_fds.remove(stream) 46 | global_buf.append(char) 47 | line_buf.append(char) 48 | if char == '\n': 49 | _test_log(''.join(line_buf).rstrip('\n')) 50 | line_buf = [] 51 | return line_buf 52 | 53 | 54 | def _poll(stdout, stderr): 55 | stdoutbuf = [] 56 | stdoutbuf_line = [] 57 | stderrbuf = [] 58 | stderrbuf_line = [] 59 | read_fds = [stdout, stderr] 60 | x_fds = [stdout, stderr] 61 | while read_fds: 62 | rlist, _, _ = select.select(read_fds, [], x_fds) 63 | if rlist: 64 | for stream in rlist: 65 | if stream == stdout: 66 | stdoutbuf_line = _process_stream(stream, read_fds, stdoutbuf, stdoutbuf_line) 67 | if stream == stderr: 68 | stderrbuf_line = _process_stream(stream, read_fds, stderrbuf, stderrbuf_line) 69 | return (''.join(stdoutbuf), ''.join(stderrbuf)) 70 | 71 | 72 | def _run_command(cmd, cwd): 73 | process = subprocess.Popen(cmd, 74 | stdout=subprocess.PIPE, 75 | stderr=subprocess.PIPE, 76 | shell=True, cwd=cwd) 77 | (stdout, stderr) = _poll(process.stdout, process.stderr) 78 | returncode = process.wait() 79 | if returncode != 0: 80 | _log(" stdout: " + stdout) 81 | _log(" stderr: " + stderr) 82 | _log(" result: " + str(returncode)) 83 | return (stdout, stderr, returncode) 84 | 85 | 86 | def _wait_until(predicate, timeoutSeconds, delaySeconds=0.25, *args, **kwargs): 87 | """ 88 | Calls a predicate repeatedly, up until a specified number of seconds, 89 | until it returns true, throws an error if true is never returned by the predicate. 90 | :param predicate: the predicate to evaluate 91 | :param timeoutSeconds: seconds before timing out 92 | :param delaySeconds: seconds between evaluations 93 | """ 94 | mustend = time.time() + timeoutSeconds 95 | while time.time() < mustend: 96 | try: 97 | if predicate(*args, **kwargs): return 98 | except Exception: 99 | pass 100 | time.sleep(delaySeconds) 101 | 102 | print("Condition not met within " + str(timeoutSeconds)) 103 | raise Exception("Condition not met within " + str(timeoutSeconds)) 104 | 105 | 106 | def _utf_encode_list(list): 107 | return [s.encode("UTF8") for s in list] 108 | 109 | 110 | def _terraform(action, vars=None): 111 | if vars == None: 112 | vars = "" 113 | else: 114 | vars += " " 115 | (stdout, _, returncode) = _run_command("terraform " + action + " " + vars, ROOT_DIR) 116 | if returncode != 0: 117 | _log("Error running terraform") 118 | raise Exception("Error running terraform") 119 | return stdout 120 | 121 | 122 | def _check_env(): 123 | pass 124 | 125 | 126 | def _handle_args(): 127 | 128 | parser = argparse.ArgumentParser( 129 | formatter_class=argparse.RawDescriptionHelpFormatter, 130 | epilog=""" 131 | description: 132 | This script allows for automated testing of Terraform Kubernetes 133 | installation configurations in the ./tests/resources/configs/ 134 | directory or by a custom terfaform.tfvars file. In any event, 135 | after applying the configuration(s), the script performs some 136 | basic tests on the K8s cluster including that the worker nodes are 137 | healthy and that a simple application can be deployed and accessed 138 | (assuming control_plane_subnet_access is public). After the 139 | validations are complete, it (optionally) tears down the configuration. 140 | 141 | examples: 142 | - ./create/runner.py 143 | 144 | Applies, tests, and tears down _all_ the example K8s cluster configurations defined in ./tests/resources/configs/*.tfvars 145 | 146 | - ./create/runner.py --tfvars-file /Users/you/configs/one-off-cluster.tfvars 147 | 148 | Applies, tests, and tears down the K8s cluster configuration defined by the custom one-off-cluster.tfvars 149 | 150 | - ./create/runner.py --no-destroy --tfvars-file /Users/you/configs/one-off-cluster.tfvars 151 | 152 | Applies and tests, the K8s cluster configuration defined in one-off-cluster.tfvars, but does not destroy it when complete 153 | """ 154 | ) 155 | 156 | parser.add_argument('--no-create', 157 | help='Disable the creation of the test infrastructure', 158 | action='store_true', 159 | default=False) 160 | parser.add_argument('--no-destroy', 161 | help='If we are creating the test infrastructure, then leave it up', 162 | action='store_true', 163 | default=False) 164 | parser.add_argument('--tfvars-file', 165 | dest='tfvars_file', 166 | help='Use a custom .tfvars file for tests (default is all .tfvars in resources/configs/)', 167 | metavar='') 168 | args = parser.parse_args() 169 | return args 170 | 171 | 172 | def _kubectl(action, exit_on_error=True): 173 | (stdout, _, returncode) = _run_command("kubectl --kubeconfig " + ROOT_DIR + "/generated/kubeconfig" + " " + action, 174 | ROOT_DIR) 175 | if exit_on_error and returncode != 0: 176 | _log("Error running kubectl") 177 | sys.exit(1) 178 | return stdout 179 | 180 | 181 | def _verifyConfig(tfvars_file, no_create=None, no_destroy=None): 182 | success = True 183 | masterPublicAddress = None 184 | try: 185 | if not no_create: 186 | _log("Creating K8s cluster from " + str(os.path.basename(tfvars_file)), as_banner=True) 187 | _terraform("init") 188 | _terraform("get") 189 | _terraform("apply -var disable_auto_retries=false -auto-approve -var-file=" + tfvars_file) 190 | 191 | # Verify expected Terraform outputs are present 192 | _log("Verifying select Terraform outputs", as_banner=True) 193 | # stdout = _terraform("output", "-json") 194 | 195 | # Figure out which IP to us (master LB or instance itself) 196 | masterLBIPOutputJSON = json.loads(_terraform("output -json master_lb_ip")) 197 | 198 | if masterLBIPOutputJSON["value"] == []: 199 | # Use the first public IP from master_public_ips 200 | masterPublicIPsOutputJSON = json.loads(_terraform("output -json master_public_ips")) 201 | masterPublicAddress = masterPublicIPsOutputJSON["value"][0] 202 | else: 203 | # Use the master LB public IP 204 | masterPublicAddress = masterLBIPOutputJSON["value"][0] 205 | 206 | masterURL = "https://" + masterPublicAddress + ":443" 207 | 208 | outputJSON = json.loads(_terraform("output -json worker_public_ips")) 209 | numWorkers = len(outputJSON["value"]) 210 | workerPublicAddressList = _utf_encode_list(outputJSON["value"]) 211 | 212 | outputJSON = json.loads(_terraform("output -json control_plane_subnet_access")) 213 | controlPlaneSubnetAccess = outputJSON["value"] 214 | 215 | _log("K8s Master URL: " + masterURL) 216 | _log("K8s Worker Public Addresses: " + str(workerPublicAddressList)) 217 | 218 | # Verify master becomes ready 219 | _log("Waiting for /healthz end-point to become available", as_banner=True) 220 | healthzOK = lambda: requests.get(masterURL + "/healthz", 221 | proxies={}, verify=False).text == "ok" 222 | _wait_until(healthzOK, 600) 223 | _log(requests.get(masterURL + "/healthz", proxies={}, verify=False).text) 224 | 225 | # Verify worker nodes become ready 226 | _log("Waiting for " + str(numWorkers) + " K8s worker nodes to become ready", as_banner=True) 227 | 228 | nodesReady = lambda: len(_kubectl("get nodes --selector=node-role.kubernetes.io/node -o name", exit_on_error=True).splitlines()) >= numWorkers 229 | _wait_until(nodesReady, 300) 230 | workerList = _kubectl("get nodes --selector=node-role.kubernetes.io/node -o name", exit_on_error=True) 231 | _log(str(workerList)) 232 | 233 | # Deploy 234 | _log("Deploying the hello service", as_banner=True) 235 | _kubectl("apply -f " + TEST_ROOT_DIR + "/resources/hello-service.yml", exit_on_error=True) 236 | time.sleep(5) 237 | _kubectl("apply -f " + TEST_ROOT_DIR + "/resources/frontend-service.yml", exit_on_error=True) 238 | 239 | # TODO poll instead of hard sleep 240 | _log("Sleeping 60 seconds to let pods initialize", as_banner=True) 241 | time.sleep(60) 242 | 243 | helloServicePort = _kubectl("get svc/hello -o jsonpath={.spec.ports[0].nodePort}", exit_on_error=True) 244 | _log("Hello service port: " + str(helloServicePort)) 245 | 246 | frontendServicePort = _kubectl("get svc/frontend -o jsonpath={.spec.ports[0].nodePort}", exit_on_error=True) 247 | _log("Frontend service port: " + str(frontendServicePort)) 248 | 249 | if controlPlaneSubnetAccess == "public": 250 | # Ping deployment 251 | _log("Pinging hello and frontend deployments for each K8s worker", as_banner=True) 252 | for workerPublicAddress in workerPublicAddressList: 253 | serviceAddressList = ["http://" + workerPublicAddress + ":" + str(helloServicePort), 254 | "http://" + workerPublicAddress + ":" + str(frontendServicePort)] 255 | for serviceAddress in serviceAddressList: 256 | _log("Checking " + serviceAddress) 257 | deploymentReady = lambda: requests.get(serviceAddress).status_code == 200 258 | _wait_until(deploymentReady, 300) 259 | 260 | except Exception, e: 261 | _log("Unexpected error:", str(e)) 262 | _log(_kubectl("get pods --all-namespaces")) 263 | _log(_kubectl("get daemonsets --all-namespaces")) 264 | traceback.print_exc() 265 | success = False 266 | finally: 267 | if masterPublicAddress != None: 268 | _log("Undeploying the hello service", as_banner=True) 269 | _kubectl("delete -f " + TEST_ROOT_DIR + "/resources/hello-service.yml", exit_on_error=False) 270 | _kubectl("delete -f " + TEST_ROOT_DIR + "/resources/frontend-service.yml", exit_on_error=False) 271 | 272 | if not no_destroy: 273 | _log("Destroying the K8s cluster from " + str(os.path.basename(tfvars_file)), as_banner=True) 274 | _terraform("destroy -force -var disable_auto_retries=true -var-file=" + tfvars_file) 275 | 276 | return success 277 | 278 | 279 | def _main(): 280 | args = _handle_args() 281 | 282 | _check_env() 283 | 284 | if not args.tfvars_file: 285 | for next_tfvars_file in os.listdir(TEST_ROOT_DIR + "/resources/configs/"): 286 | next_succcess = _verifyConfig(TEST_ROOT_DIR + "/resources/configs/" + next_tfvars_file, args.no_create, 287 | args.no_destroy) 288 | if not next_succcess: 289 | sys.exit(1) 290 | else: 291 | success = _verifyConfig(args.tfvars_file, args.no_create, args.no_destroy) 292 | if not success: 293 | sys.exit(1) 294 | 295 | 296 | if __name__ == "__main__": 297 | _main() 298 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.18.1 -------------------------------------------------------------------------------- /tests/resources/configs/public-cluster.tfvars: -------------------------------------------------------------------------------- 1 | # 2 | # CI Pipeline example cluster configuration details 3 | # 4 | 5 | label_prefix = "wercker-ci-tf-installer-" 6 | vcn_dns_name = "k8soci" 7 | domain_name = "k8soci.oraclevcn.com" 8 | control_plane_subnet_access = "public" 9 | k8s_master_lb_access = "public" 10 | master_oci_lb_enabled = "false" 11 | etcd_lb_enabled = "false" 12 | etcdShape = "VM.Standard2.2" 13 | k8sMasterShape = "VM.Standard2.1" 14 | k8sWorkerShape = "VM.Standard2.1" 15 | etcdAd1Count = "1" 16 | etcdAd2Count = "0" 17 | etcdAd3Count = "0" 18 | k8sMasterAd1Count = "1" 19 | k8sMasterAd2Count = "0" 20 | k8sMasterAd3Count = "0" 21 | k8sWorkerAd1Count = "0" 22 | k8sWorkerAd2Count = "1" 23 | k8sWorkerAd3Count = "1" 24 | etcdLBShape = "400Mbps" 25 | k8sMasterLBShape = "400Mbps" 26 | etcd_cluster_ingress = "10.0.0.0/16" 27 | etcd_ssh_ingress = "10.0.0.0/16" 28 | master_ssh_ingress = "10.0.0.0/16" 29 | worker_ssh_ingress = "10.0.0.0/16" 30 | master_https_ingress = "0.0.0.0/0" 31 | worker_nodeport_ingress = "0.0.0.0/0" 32 | -------------------------------------------------------------------------------- /tests/resources/frontend-service.yml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: frontend 5 | spec: 6 | selector: 7 | app: hello 8 | tier: frontend 9 | ports: 10 | - protocol: "TCP" 11 | port: 80 12 | targetPort: 80 13 | type: NodePort 14 | --- 15 | apiVersion: apps/v1beta1 16 | kind: Deployment 17 | metadata: 18 | name: frontend 19 | spec: 20 | replicas: 3 21 | template: 22 | metadata: 23 | labels: 24 | app: hello 25 | tier: frontend 26 | track: stable 27 | spec: 28 | containers: 29 | - name: nginx 30 | image: "gcr.io/google-samples/hello-frontend:1.0" 31 | lifecycle: 32 | preStop: 33 | exec: 34 | command: ["/usr/sbin/nginx","-s","quit"] 35 | -------------------------------------------------------------------------------- /tests/resources/hello-service.yml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: hello 5 | spec: 6 | selector: 7 | app: hello 8 | tier: backend 9 | ports: 10 | - protocol: TCP 11 | port: 80 12 | targetPort: http 13 | type: NodePort 14 | --- 15 | apiVersion: apps/v1beta1 16 | kind: Deployment 17 | metadata: 18 | name: hello 19 | spec: 20 | replicas: 15 21 | template: 22 | metadata: 23 | labels: 24 | app: hello 25 | tier: backend 26 | track: stable 27 | spec: 28 | containers: 29 | - name: hello 30 | image: "gcr.io/google-samples/hello-go-gke:1.0" 31 | ports: 32 | - name: http 33 | containerPort: 80 34 | -------------------------------------------------------------------------------- /tls/main.tf: -------------------------------------------------------------------------------- 1 | # Create a Cluster Root CA 2 | 3 | resource "tls_private_key" "root-ca" { 4 | count = "${var.ca_cert == "" ? 1 : 0}" 5 | algorithm = "RSA" 6 | rsa_bits = 2048 7 | } 8 | 9 | resource "tls_self_signed_cert" "root-ca" { 10 | count = "${var.ca_cert == "" ? 1 : 0}" 11 | key_algorithm = "RSA" 12 | private_key_pem = "${tls_private_key.root-ca.private_key_pem}" 13 | is_ca_certificate = true 14 | 15 | subject { 16 | common_name = "${var.common_name}" 17 | } 18 | 19 | allowed_uses = [ 20 | "key_encipherment", 21 | "cert_signing", 22 | "server_auth", 23 | "client_auth", 24 | ] 25 | 26 | validity_period_hours = "${var.validity_period_hours}" 27 | } 28 | 29 | # Kubernetes API Server Keypair 30 | 31 | resource "tls_private_key" "api-server" { 32 | count = "${var.api_server_private_key == "" ? 1 : 0}" 33 | algorithm = "RSA" 34 | rsa_bits = 2048 35 | } 36 | 37 | resource "tls_cert_request" "api-server" { 38 | count = "${var.api_server_cert == "" ? 1 : 0}" 39 | key_algorithm = "RSA" 40 | private_key_pem = "${tls_private_key.api-server.private_key_pem}" 41 | 42 | # TODO DNS name of LB 43 | dns_names = "${concat(list(var.k8s-serviceip), 44 | list( 45 | "localhost", 46 | "kubernetes", 47 | "kubernetes.default", 48 | "kubernetes.default.svc", 49 | "kubernetes.default.svc.cluster.local" 50 | ))}" 51 | ip_addresses = ["${distinct(list( 52 | "${var.master_lb_public_ip}", 53 | "${var.k8s-serviceip}", 54 | "127.0.0.1" 55 | ))}"] 56 | 57 | # system:masters group 58 | subject { 59 | common_name = "*" 60 | organization = "system:masters" 61 | } 62 | } 63 | 64 | resource "tls_locally_signed_cert" "api-server" { 65 | count = "${var.api_server_cert == "" ? 1 : 0}" 66 | cert_request_pem = "${tls_cert_request.api-server.cert_request_pem}" 67 | ca_key_algorithm = "RSA" 68 | ca_private_key_pem = "${tls_private_key.root-ca.private_key_pem}" 69 | ca_cert_pem = "${tls_self_signed_cert.root-ca.cert_pem}" 70 | validity_period_hours = "${var.validity_period_hours}" 71 | 72 | allowed_uses = [ 73 | "key_encipherment", 74 | "server_auth", 75 | "client_auth", 76 | "digital_signature", 77 | ] 78 | } 79 | 80 | # Instance SSH Key Pairs 81 | 82 | resource "tls_private_key" "ssh" { 83 | lifecycle { 84 | create_before_destroy = true 85 | } 86 | 87 | algorithm = "RSA" 88 | count = "${var.ssh_private_key == "" ? 1 : 0}" 89 | rsa_bits = 2048 90 | } 91 | 92 | resource "random_id" "token-auth" { 93 | count = "${var.api_server_admin_token == "" ? 1 : 0}" 94 | byte_length = 16 95 | } 96 | -------------------------------------------------------------------------------- /tls/outputs.tf: -------------------------------------------------------------------------------- 1 | output "root_ca_pem" { 2 | value = "${var.ca_cert == "" ? join(" ", tls_self_signed_cert.root-ca.*.cert_pem) : var.ca_cert}" 3 | } 4 | 5 | output "root_ca_key" { 6 | value = "${var.ca_key == "" ? join(" ", tls_private_key.root-ca.*.private_key_pem) : var.ca_key}" 7 | } 8 | 9 | output "api_server_private_key_pem" { 10 | value = "${var.api_server_private_key == "" ? join(" ", tls_private_key.api-server.*.private_key_pem) : var.api_server_private_key}" 11 | } 12 | 13 | output "api_server_cert_pem" { 14 | value = "${var.api_server_cert == "" ? join(" ", tls_locally_signed_cert.api-server.*.cert_pem) : var.api_server_cert}" 15 | } 16 | 17 | output "api_server_admin_token" { 18 | value = "${var.api_server_admin_token == "" ? join(" ", random_id.token-auth.*.hex) : var.api_server_admin_token}" 19 | } 20 | 21 | output "ssh_private_key" { 22 | value = "${var.ssh_private_key == "" ? join(" ", tls_private_key.ssh.*.private_key_pem) : var.ssh_private_key}" 23 | } 24 | 25 | output "ssh_public_key_openssh" { 26 | value = "${var.ssh_public_key_openssh == "" ? join(" ", tls_private_key.ssh.*.public_key_openssh) : var.ssh_public_key_openssh}" 27 | } 28 | -------------------------------------------------------------------------------- /tls/variables.tf: -------------------------------------------------------------------------------- 1 | # valid for 1000 days 2 | variable "validity_period_hours" { 3 | default = 24000 4 | } 5 | 6 | variable "ca_cert" { 7 | description = "CA certificate in PEM format(generated if left blank)" 8 | default = "" 9 | } 10 | 11 | variable "ca_key" { 12 | description = "CA private key in PEM format (generated if left blank)" 13 | default = "" 14 | } 15 | 16 | variable "api_server_private_key" { 17 | description = "API Server private key in PEM format (generated if left blank)" 18 | default = "" 19 | } 20 | 21 | variable "api_server_cert" { 22 | description = "API Server cert in PEM format (generated if left blank)" 23 | default = "" 24 | } 25 | 26 | variable "api_server_admin_token" { 27 | description = "admin user's bearer token for API server (generated if left blank)" 28 | type = "string" 29 | default = "" 30 | } 31 | 32 | variable "common_name" { 33 | default = "kube-ca" 34 | } 35 | 36 | variable "k8s-serviceip" { 37 | default = "10.21.0.1" 38 | } 39 | 40 | variable "master_lb_public_ip" {} 41 | 42 | variable "ssh_private_key" { 43 | description = "SSH private key for instances (generated if left blank)" 44 | default = "" 45 | } 46 | 47 | variable "ssh_public_key_openssh" { 48 | description = "SSH public key in OpenSSH authorized_keys format for instances (generated if left blank)" 49 | default = "" 50 | } 51 | -------------------------------------------------------------------------------- /wercker.yml: -------------------------------------------------------------------------------- 1 | no-response-timeout: 15 2 | command-timeout: 60 3 | 4 | init: 5 | box: 6 | id: alpine:3.7 7 | cmd: /bin/sh 8 | steps: 9 | - script: 10 | name: init 11 | code: echo "Init" 12 | 13 | build: 14 | box: 15 | id: alpine:3.7 16 | cmd: /bin/sh 17 | steps: 18 | - script: 19 | name: install curl and unzip 20 | code: apk upgrade && apk update && apk add curl && apk add unzip 21 | - script: 22 | name: download Terraform binary 23 | code: | 24 | curl -LO --retry 3 https://releases.hashicorp.com/terraform/0.11.1/terraform_0.11.1_linux_amd64.zip 25 | unzip terraform_0.11.1_linux_amd64.zip 26 | chmod +x ./terraform 27 | mv terraform /usr/bin/ 28 | - script: 29 | name: download and configure Terraform OCI Provider 30 | code: | 31 | curl -LO --retry 3 https://github.com/oracle/terraform-provider-oci/releases/download/v2.0.6/linux.tar.gz 32 | tar -xzvf linux.tar.gz -C / 33 | echo 'providers { oci = "/linux_amd64/terraform-provider-oci_v2.0.6" }' > ~/.terraformrc 34 | - script: 35 | name: run terraform validate for static validation 36 | code: | 37 | export TF_VAR_private_key_path=/tmp/bmcs_api_key.pem 38 | echo -e "$TF_VAR_private_key" > $TF_VAR_private_key_path 39 | terraform init 40 | terraform validate 41 | 42 | deploy: 43 | box: 44 | id: python:2.7 45 | steps: 46 | - install-packages: 47 | packages: unzip 48 | - pip-install: 49 | requirements_file: "tests/requirements.txt" 50 | - script: 51 | name: download Terraform binary 52 | code: | 53 | curl -LO --retry 3 https://releases.hashicorp.com/terraform/0.11.1/terraform_0.11.1_linux_amd64.zip 54 | unzip terraform_0.11.1_linux_amd64.zip 55 | chmod +x ./terraform 56 | mv terraform /usr/bin/ 57 | - script: 58 | name: download and configure Terraform OCI Provider 59 | code: | 60 | curl -LO --retry 3 https://github.com/oracle/terraform-provider-oci/releases/download/v2.0.6/linux.tar.gz 61 | tar -xzvf linux.tar.gz -C / 62 | echo 'providers { oci = "/linux_amd64/terraform-provider-oci_v2.0.6" }' > ~/.terraformrc 63 | - script: 64 | name: download kubectl binary 65 | code: | 66 | curl -LO --retry 3 https://storage.googleapis.com/kubernetes-release/release/v1.8.5/bin/linux/amd64/kubectl 67 | chmod +x ./kubectl 68 | mv kubectl /usr/bin/ 69 | - script: 70 | name: run cluster creation tests 71 | code: | 72 | rm -rf .terraform/ 73 | export TF_VAR_private_key_path=/tmp/bmcs_api_key.pem 74 | echo -e "$TF_VAR_private_key" > $TF_VAR_private_key_path 75 | terraform init 76 | terraform validate 77 | cd tests/ 78 | export EXTERNAL_IP=$(curl -s -m 10 http://whatismyip.akamai.com/) 79 | find ./resources/configs -type f -exec sed -i "s/0.0.0.0\/0/$EXTERNAL_IP\/32/g" {} + 80 | python2.7 create/runner.py 2>&1 | tee createtests.log ; test ${PIPESTATUS[0]} -eq 0 81 | 82 | after-steps: 83 | - script: 84 | name: print cluster creation test log 85 | code: | 86 | cat tests/createtests.log || true 87 | - script: 88 | name: perform additional cleanup after tests 89 | code: | 90 | export TF_VAR_private_key_path=/tmp/bmcs_api_key.pem 91 | echo -e "$TF_VAR_private_key" > $TF_VAR_private_key_path 92 | terraform destroy -force || true 93 | terraform destroy -refresh=true -force || true 94 | terraform destroy -force -target=module.instances-k8smaster-ad1 || true 95 | terraform destroy -force -target=module.instances-k8smaster-ad2 || true 96 | terraform destroy -force -target=module.instances-k8smaster-ad3 || true 97 | terraform destroy -force -target=module.instances-k8sworker-ad1 || true 98 | terraform destroy -force -target=module.instances-k8sworker-ad2 || true 99 | terraform destroy -force -target=module.instances-k8sworker-ad3 || true 100 | terraform destroy -force -target=module.instances-etcd-ad1 || true 101 | terraform destroy -force -target=module.instances-etcd-ad2 || true 102 | terraform destroy -force -target=module.instances-etcd-ad3 || true 103 | --------------------------------------------------------------------------------