├── .drone.yml ├── LICENSE ├── README.md ├── clusters ├── a1-k8s-inf │ └── cluster.yml ├── a1-k8s-lab │ └── cluster.yml ├── a1-k8s-npd │ └── cluster.yml ├── a1-rancher-dev │ ├── cluster.yml │ └── rancher-values.yaml ├── a1-rancher-lab │ ├── cluster.yml │ └── rancher-values.yaml └── template │ ├── cluster.yml │ └── rancher-values.yaml ├── run.sh └── s3-template └── creds /.drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: kubernetes 4 | name: Rancher-Cluster-Builder 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | concurrency: 11 | limit: 1 12 | 13 | steps: 14 | - name: Builder 15 | image: supporttools/rancher-cluster-builder:latest 16 | pull: true 17 | environment: 18 | SSH_KEY: 19 | from_secret: ssh_key 20 | AWS_ACCESS_KEY_ID: 21 | from_secret: S3_ACCESSKEY 22 | AWS_SECRET_ACCESS_KEY: 23 | from_secret: S3_SECRETKEY 24 | AWS_DEFAULT_REGION: 25 | from_secret: S3_REGION 26 | S3_BUCKET: 27 | from_secret: S3_BUCKET 28 | S3_ENDPOINT: 29 | from_secret: S3_ENDPOINT 30 | commands: 31 | - bash ./run.sh 32 | 33 | node_selector: 34 | kubernetes.io/arch: amd64 35 | kubernetes.io/os: linux 36 | drone-runner: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://drone.support.tools/api/badges/SupportTools/rancher-cluster-builder/status.svg)](https://drone.support.tools/SupportTools/rancher-cluster-builder) 2 | [![Twitter](https://img.shields.io/twitter/follow/cube8021?style=social&logo=twitter)](https://twitter.com/cube8021) 3 | [![Pulls](https://img.shields.io/docker/pulls/supporttools/rancher-cluster-builder.svg)](https://hub.docker.com/r/supporttools/rancher-cluster-builder) 4 | ![Image](https://img.shields.io/docker/image-size/supporttools/rancher-cluster-builder) 5 | 6 | Rancher Clusters Builder 7 | ======================== 8 | 9 | The rancher-clusters-builder is designed to manage RKE clusters (mainly Rancher local clusters) using a drone as the CICD pipeline, GitHub as the code repository, and Wasabisys's S3 storage for storing the artifacts from the build process. 10 | 11 | ## Required Drone secrets 12 | - S3_ACCESSKEY 13 | This should be the plaintext access key for accessing the S3 bucket. 14 | ``` 15 | Example: AKIAIOSFODNN7EXAMPLE 16 | ``` 17 | 18 | - S3_SECRETKEY 19 | This should be the plaintext secret key for accessing the S3 bucket. 20 | ``` 21 | Example: wJalrXUtnFEMI/K7MDENG/bPxRfiCEXAMPLEKEY 22 | ``` 23 | 24 | - S3_BUCKET 25 | This should be the S3 bucket name for storing the artifacts (cluster.rkestate, kube_config_cluster.yml, tls.crt, tls.key) 26 | ``` 27 | Example: rancher-clusters 28 | ``` 29 | 30 | - S3_ENDPOINT 31 | This should be the S3 API endpoint. This is optional is using AWS S3. 32 | ``` 33 | Example: https://s3.us-central-1.wasabisys.com 34 | ``` 35 | 36 | - S3_REGION 37 | This should be the region for the S3 bucket. 38 | ``` 39 | Example: us-east-1 40 | ``` 41 | 42 | - ssh_key 43 | This should be the plaintext SSH private key for accessing the RKE nodes. 44 | ``` 45 | Example: `-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1r......\n-----END OPENSSH PRIVATE KEY-----` 46 | ``` 47 | 48 | ## Adding a cluster 49 | Before starting, we'll assume that the Servers in this cluster have been built and have docker installed. To add a cluster to the pipeline, you must create a folder in `clusters`; this folder name will be the cluster's name. So please try not to use any control characters or spaces in the name. Inside that folder, we'll need to create the cluster.yml using the template file provided in `clusters/template.` To protect for storing passwords in git; we use the file `creds` (An example can be found in `s3-template/creds`) to find and replace the values. Note, the creds file should be stored in S3 in the folder `clusters/ClusterName/creds.` Once the files have been created and committed to the repository. The drone build runs a basic test on the main build script, `run.sh`. Then to build the cluster, we'll promote the drone build using the following options. 50 | 51 | - Action: Promote 52 | - Environment: (This can be anything but it must be something) 53 | - Parameters: 54 | - Action=cluster_up 55 | - Cluster=ClusterNameHere 56 | 57 | Once this build is started, the tool will verify the required files are in the right place. It will then handle the process of building the RKE cluster using the command `rke up`. Once the cluster has been created, the files `cluster.rkestate` and `kube_config_cluster.yml` will be synced up to S3 as needed for the next time we do a `rke up.` 58 | 59 | ## Upgrading k8s on a cluster 60 | We will need to edit the value for `kubernetes_version` in the cluster.yml for the cluster we would like to update. The supported Kubernetes versions can be found at [here](https://raw.githubusercontent.com/rancher/kontainer-driver-metadata/dev-v2.5/data/data.json) 61 | 62 | Once the changes have been committed to the repository, we'll promote the drone build using the following options. 63 | 64 | - Action: Promote 65 | - Environment: (This can be anything but it must be something) 66 | - Parameters: 67 | - Action=cluster_up 68 | - Cluster=ClusterNameHere 69 | 70 | ## Installing/Upgrading Rancher on an RKE cluster 71 | We'll assume that the RKE cluster is being managed by this tool. To install/upgrade Rancher on a cluster, we'll need to create the file `rancher-values.yaml` in the cluster folder that we want to deploy to. All the install flags can be found [here](https://rancher.com/docs/rancher/v2.x/en/installation/install-rancher-on-k8s/) Note: If you're going to bring your certificate, you'll need to upload the certificate and key to the S3 bucket under the cluster folder with the file names tls.crt and tls.key. 72 | 73 | Once the changes have been committed to the repository, we'll promote the drone build using the following options. 74 | 75 | - Action: Promote 76 | - Environment: (This can be anything but it must be something) 77 | - Parameters: 78 | - Action=rancher_up 79 | - Cluster=ClusterNameHere 80 | 81 | ## Applying a rolling reboot to a cluster 82 | To do a safe rolling reboot of an RKE cluster, we'll cordon and drain the node. Then reboot the server and wait for the node to return. Once the node has returned, we'll uncordon it and run an RKE up to verify the cluster is healthy. Once all that is done, we'll move on to the next node in the cluster. Note: This process has many periods of sleep sections and is designed to be as safe as possible. 83 | -------------------------------------------------------------------------------- /clusters/a1-k8s-inf/cluster.yml: -------------------------------------------------------------------------------- 1 | local_cluster_auth_endpoint: 2 | ca_certs: |- 3 | -----BEGIN CERTIFICATE----- 4 | MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG 5 | A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv 6 | b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw 7 | MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i 8 | YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT 9 | aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ 10 | jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp 11 | xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp 12 | 1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG 13 | snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ 14 | U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 15 | 9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E 16 | BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B 17 | AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz 18 | yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE 19 | 38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP 20 | AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad 21 | DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME 22 | HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== 23 | -----END CERTIFICATE----- 24 | enabled: true 25 | fqdn: a1-k8s-inf-k8s.support.tools 26 | 27 | nodes: 28 | - address: 208.86.161.87 29 | user: root 30 | hostname_override: a1ubk8si01 31 | internal_address: 172.27.2.33 32 | role: [controlplane,worker,etcd] 33 | - address: 172.27.2.34 34 | user: root 35 | hostname_override: a1ubk8si02 36 | internal_address: 172.27.2.34 37 | role: [controlplane,worker,etcd] 38 | - address: 172.27.2.35 39 | user: root 40 | hostname_override: a1ubk8si03 41 | internal_address: 172.27.2.35 42 | role: [controlplane,worker,etcd] 43 | - address: 172.27.2.36 44 | user: root 45 | hostname_override: a1ublbi01 46 | internal_address: 172.27.2.36 47 | role: [worker] 48 | labels: 49 | app: ingress 50 | - address: 172.27.2.37 51 | user: root 52 | hostname_override: a1ublbi02 53 | internal_address: 172.27.2.37 54 | role: [worker] 55 | labels: 56 | app: ingress 57 | - address: 172.27.2.44 58 | user: root 59 | hostname_override: a1ublongi01 60 | internal_address: 172.27.2.44 61 | role: [worker] 62 | labels: 63 | app: longhorn 64 | - address: 172.27.2.45 65 | user: root 66 | hostname_override: a1ublongi02 67 | internal_address: 172.27.2.45 68 | role: [worker] 69 | labels: 70 | app: longhorn 71 | - address: 172.27.2.46 72 | user: root 73 | hostname_override: a1ublongi03 74 | internal_address: 172.27.2.46 75 | role: [worker] 76 | labels: 77 | app: longhorn 78 | - address: 172.27.2.47 79 | user: root 80 | hostname_override: a1ublongi04 81 | internal_address: 172.27.2.47 82 | role: [worker] 83 | labels: 84 | app: longhorn 85 | - address: 172.27.2.48 86 | user: root 87 | hostname_override: a1ublongi05 88 | internal_address: 172.27.2.48 89 | role: [worker] 90 | labels: 91 | app: longhorn 92 | 93 | ingress: 94 | provider: nginx 95 | node_selector: 96 | app: ingress 97 | tolerations: 98 | - key: "node.kubernetes.io/unreachable" 99 | operator: "Exists" 100 | effect: "NoExecute" 101 | tolerationseconds: 300 102 | - key: "node.kubernetes.io/not-ready" 103 | operator: "Exists" 104 | effect: "NoExecute" 105 | tolerationseconds: 300 106 | 107 | services: 108 | etcd: 109 | backup_config: 110 | enabled: true # enables recurring etcd snapshots 111 | interval_hours: 12 # time increment between snapshots 112 | retention: 7 # time in days before snapshot purge 113 | s3backupconfig: 114 | access_key: "s3backupconfig.access_key" 115 | secret_key: "s3backupconfig.secret_key" 116 | bucket_name: "s3backupconfig.bucket_name" 117 | folder: "s3backupconfig.folder" 118 | endpoint: "s3backupconfig.endpoint" 119 | region: "s3backupconfig.region" 120 | kube-api: 121 | service_cluster_ip_range: 10.123.0.0/16 122 | kube-controller: 123 | cluster_cidr: 10.122.0.0/16 124 | service_cluster_ip_range: 10.123.0.0/16 125 | kubelet: 126 | cluster_dns_server: 10.123.0.10 127 | cluster_domain: a1-k8s-inf.local 128 | 129 | cloud_provider: 130 | name: vsphere 131 | vsphereCloudProvider: 132 | disk: {} 133 | global: 134 | insecure-flag: true 135 | soap-roundtrip-count: 0 136 | network: {} 137 | virtual_center: 138 | cloud_provider.virtual_center: 139 | datacenters: cloud_provider.datacenters 140 | port: cloud_provider.port 141 | soap-roundtrip-count: 0 142 | user: cloud_provider.username 143 | password: cloud_provider.password 144 | workspace: 145 | datacenter: cloud_provider.datacenters 146 | default-datastore: cloud_provider.default-datastore 147 | folder: cloud_provider.folder 148 | server: cloud_provider.virtual_center 149 | 150 | dns: 151 | provider: coredns 152 | upstreamnameservers: 153 | - 172.27.2.23 154 | - 172.27.2.24 155 | 156 | kubernetes_version: v1.19.7-rancher1-1 157 | 158 | addons_include: 159 | - rancher_import 160 | -------------------------------------------------------------------------------- /clusters/a1-k8s-lab/cluster.yml: -------------------------------------------------------------------------------- 1 | local_cluster_auth_endpoint: 2 | ca_certs: |- 3 | -----BEGIN CERTIFICATE----- 4 | MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG 5 | A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv 6 | b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw 7 | MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i 8 | YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT 9 | aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ 10 | jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp 11 | xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp 12 | 1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG 13 | snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ 14 | U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 15 | 9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E 16 | BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B 17 | AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz 18 | yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE 19 | 38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP 20 | AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad 21 | DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME 22 | HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== 23 | -----END CERTIFICATE----- 24 | enabled: true 25 | fqdn: a1-k8s-labs-k8s.support.tools 26 | 27 | nodes: 28 | - address: 172.27.7.31 29 | user: root 30 | hostname_override: a1ubk8sl01 31 | internal_address: 172.27.7.31 32 | role: [controlplane,worker,etcd] 33 | - address: 172.27.7.32 34 | user: root 35 | hostname_override: a1ubk8sl02 36 | internal_address: 172.27.7.32 37 | role: [controlplane,worker,etcd] 38 | - address: 172.27.7.33 39 | user: root 40 | hostname_override: a1ubk8sl03 41 | internal_address: 172.27.7.33 42 | role: [controlplane,worker,etcd] 43 | 44 | ingress: 45 | provider: nginx 46 | 47 | services: 48 | etcd: 49 | backup_config: 50 | enabled: true # enables recurring etcd snapshots 51 | interval_hours: 12 # time increment between snapshots 52 | retention: 6 # time in days before snapshot purge 53 | kube-api: 54 | service_cluster_ip_range: 10.153.0.0/16 55 | kube-controller: 56 | cluster_cidr: 10.152.0.0/16 57 | service_cluster_ip_range: 10.153.0.0/16 58 | kubelet: 59 | cluster_dns_server: 10.153.0.10 60 | cluster_domain: a1-k8s-lab.local 61 | 62 | dns: 63 | provider: coredns 64 | upstreamnameservers: 65 | - 172.27.2.23 66 | - 172.27.2.24 67 | 68 | kubernetes_version: v1.19.7-rancher1-1 69 | 70 | addons_include: 71 | - rancher_import 72 | -------------------------------------------------------------------------------- /clusters/a1-k8s-npd/cluster.yml: -------------------------------------------------------------------------------- 1 | local_cluster_auth_endpoint: 2 | ca_certs: |- 3 | -----BEGIN CERTIFICATE----- 4 | MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG 5 | A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv 6 | b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw 7 | MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i 8 | YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT 9 | aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ 10 | jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp 11 | xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp 12 | 1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG 13 | snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ 14 | U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 15 | 9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E 16 | BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B 17 | AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz 18 | yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE 19 | 38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP 20 | AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad 21 | DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME 22 | HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== 23 | -----END CERTIFICATE----- 24 | enabled: true 25 | fqdn: a1-k8s-npd-k8s.support.tools 26 | 27 | nodes: 28 | - address: 172.27.6.21 29 | user: root 30 | hostname_override: a1ubk8sd01 31 | internal_address: 172.27.6.21 32 | role: [controlplane,worker,etcd] 33 | - address: 172.27.6.22 34 | user: root 35 | hostname_override: a1ubk8sd02 36 | internal_address: 172.27.6.22 37 | role: [controlplane,worker,etcd] 38 | - address: 172.27.6.23 39 | user: root 40 | hostname_override: a1ubk8sd03 41 | internal_address: 172.27.6.23 42 | role: [controlplane,worker,etcd] 43 | 44 | ingress: 45 | provider: nginx 46 | 47 | services: 48 | etcd: 49 | backup_config: 50 | enabled: true # enables recurring etcd snapshots 51 | interval_hours: 12 # time increment between snapshots 52 | retention: 7 # time in days before snapshot purge 53 | s3backupconfig: 54 | access_key: "s3backupconfig.access_key" 55 | secret_key: "s3backupconfig.secret_key" 56 | bucket_name: "s3backupconfig.bucket_name" 57 | folder: "s3backupconfig.folder" 58 | endpoint: "s3backupconfig.endpoint" 59 | region: "s3backupconfig.region" 60 | kube-api: 61 | service_cluster_ip_range: 10.113.0.0/16 62 | kube-controller: 63 | cluster_cidr: 10.112.0.0/16 64 | service_cluster_ip_range: 10.113.0.0/16 65 | kubelet: 66 | cluster_dns_server: 10.113.0.10 67 | cluster_domain: a1-k8s-npd.local 68 | 69 | cloud_provider: 70 | name: vsphere 71 | vsphereCloudProvider: 72 | disk: {} 73 | global: 74 | insecure-flag: true 75 | soap-roundtrip-count: 0 76 | network: {} 77 | virtual_center: 78 | cloud_provider.virtual_center: 79 | datacenters: cloud_provider.datacenters 80 | port: cloud_provider.port 81 | soap-roundtrip-count: 0 82 | user: cloud_provider.username 83 | password: cloud_provider.password 84 | workspace: 85 | datacenter: cloud_provider.datacenters 86 | default-datastore: cloud_provider.default-datastore 87 | folder: cloud_provider.folder 88 | server: cloud_provider.virtual_center 89 | 90 | dns: 91 | provider: coredns 92 | upstreamnameservers: 93 | - 172.27.2.23 94 | - 172.27.2.24 95 | 96 | kubernetes_version: v1.19.7-rancher1-1 97 | -------------------------------------------------------------------------------- /clusters/a1-rancher-dev/cluster.yml: -------------------------------------------------------------------------------- 1 | nodes: 2 | - address: 172.27.6.21 3 | user: root 4 | hostname_override: a1ubrancherd01 5 | internal_address: 172.27.6.21 6 | role: [controlplane,worker,etcd] 7 | - address: 172.27.6.22 8 | user: root 9 | hostname_override: a1ubrancherd02 10 | internal_address: 172.27.6.22 11 | role: [controlplane,worker,etcd] 12 | - address: 172.27.6.23 13 | user: root 14 | hostname_override: a1ubrancherd03 15 | internal_address: 172.27.6.23 16 | role: [controlplane,worker,etcd] 17 | 18 | ingress: 19 | provider: nginx 20 | 21 | services: 22 | etcd: 23 | backup_config: 24 | enabled: true # enables recurring etcd snapshots 25 | interval_hours: 12 # time increment between snapshots 26 | retention: 7 # time in days before snapshot purge 27 | s3backupconfig: 28 | access_key: "s3backupconfig.access_key" 29 | secret_key: "s3backupconfig.secret_key" 30 | bucket_name: "s3backupconfig.bucket_name" 31 | folder: "s3backupconfig.folder" 32 | endpoint: "s3backupconfig.endpoint" 33 | region: "s3backupconfig.region" 34 | 35 | cloud_provider: 36 | name: vsphere 37 | vsphereCloudProvider: 38 | disk: {} 39 | global: 40 | insecure-flag: true 41 | soap-roundtrip-count: 0 42 | network: {} 43 | virtual_center: 44 | cloud_provider.virtual_center: 45 | datacenters: cloud_provider.datacenters 46 | port: cloud_provider.port 47 | soap-roundtrip-count: 0 48 | user: cloud_provider.username 49 | password: cloud_provider.password 50 | workspace: 51 | datacenter: cloud_provider.datacenters 52 | default-datastore: cloud_provider.default-datastore 53 | folder: cloud_provider.folder 54 | server: cloud_provider.virtual_center 55 | 56 | dns: 57 | provider: coredns 58 | upstreamnameservers: 59 | - 172.27.2.23 60 | - 172.27.2.24 61 | 62 | kubernetes_version: v1.20.5-rancher1-1 63 | -------------------------------------------------------------------------------- /clusters/a1-rancher-dev/rancher-values.yaml: -------------------------------------------------------------------------------- 1 | antiAffinity: required 2 | auditLog: 3 | level: 2 4 | hostname: a1-rancher-dev.support.tools 5 | ingress: 6 | tls: 7 | source: secret 8 | rancher_verison: 2.5.7 9 | rancher_chart: rancher-latest/rancher 10 | -------------------------------------------------------------------------------- /clusters/a1-rancher-lab/cluster.yml: -------------------------------------------------------------------------------- 1 | local_cluster_auth_endpoint: 2 | ca_certs: |- 3 | -----BEGIN CERTIFICATE----- 4 | MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG 5 | A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv 6 | b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw 7 | MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i 8 | YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT 9 | aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ 10 | jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp 11 | xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp 12 | 1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG 13 | snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ 14 | U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 15 | 9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E 16 | BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B 17 | AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz 18 | yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE 19 | 38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP 20 | AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad 21 | DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME 22 | HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== 23 | -----END CERTIFICATE----- 24 | enabled: true 25 | fqdn: a1-rancher-lab-k8s.support.tools 26 | 27 | nodes: 28 | - address: 172.27.7.21 29 | user: root 30 | hostname_override: a1ubrancherl01 31 | internal_address: 172.27.7.21 32 | role: [controlplane,worker,etcd] 33 | - address: 172.27.7.22 34 | user: root 35 | hostname_override: a1ubrancherl02 36 | internal_address: 172.27.7.22 37 | role: [controlplane,worker,etcd] 38 | - address: 172.27.7.23 39 | user: root 40 | hostname_override: a1ubrancherl03 41 | internal_address: 172.27.7.23 42 | role: [controlplane,worker,etcd] 43 | 44 | ingress: 45 | provider: nginx 46 | 47 | services: 48 | etcd: 49 | backup_config: 50 | enabled: true # enables recurring etcd snapshots 51 | interval_hours: 12 # time increment between snapshots 52 | retention: 7 # time in days before snapshot purge 53 | s3backupconfig: 54 | access_key: "s3backupconfig.access_key" 55 | secret_key: "s3backupconfig.secret_key" 56 | bucket_name: "s3backupconfig.bucket_name" 57 | folder: "s3backupconfig.folder" 58 | endpoint: "s3backupconfig.endpoint" 59 | region: "s3backupconfig.region" 60 | 61 | cloud_provider: 62 | name: vsphere 63 | vsphereCloudProvider: 64 | disk: {} 65 | global: 66 | insecure-flag: true 67 | soap-roundtrip-count: 0 68 | network: {} 69 | virtual_center: 70 | cloud_provider.virtual_center: 71 | datacenters: cloud_provider.datacenters 72 | port: cloud_provider.port 73 | soap-roundtrip-count: 0 74 | user: cloud_provider.username 75 | password: cloud_provider.password 76 | workspace: 77 | datacenter: cloud_provider.datacenters 78 | default-datastore: cloud_provider.default-datastore 79 | folder: cloud_provider.folder 80 | server: cloud_provider.virtual_center 81 | 82 | dns: 83 | provider: coredns 84 | upstreamnameservers: 85 | - 172.27.2.23 86 | - 172.27.2.24 87 | 88 | kubernetes_version: v1.20.5-rancher1-1 89 | -------------------------------------------------------------------------------- /clusters/a1-rancher-lab/rancher-values.yaml: -------------------------------------------------------------------------------- 1 | antiAffinity: required 2 | auditLog: 3 | level: 2 4 | hostname: a1-rancher-lab.support.tools 5 | ingress: 6 | tls: 7 | source: secret 8 | rancher_verison: 2.5.7 9 | rancher_chart: rancher-latest/rancher 10 | -------------------------------------------------------------------------------- /clusters/template/cluster.yml: -------------------------------------------------------------------------------- 1 | nodes: 2 | - address: 1.1.1.1 3 | user: root 4 | hostname_override: node01 5 | role: [controlplane,worker,etcd] 6 | - address: 2.2.2.2 7 | user: root 8 | hostname_override: node02 9 | role: [controlplane,worker,etcd] 10 | - address: 3.3.3.3 11 | user: root 12 | hostname_override: node03 13 | role: [controlplane,worker,etcd] 14 | 15 | ingress: 16 | provider: nginx 17 | 18 | services: 19 | etcd: 20 | backup_config: 21 | enabled: true # enables recurring etcd snapshots 22 | interval_hours: 12 # time increment between snapshots 23 | retention: 7 # time in days before snapshot purge 24 | s3backupconfig: 25 | access_key: "s3backupconfig.access_key" 26 | secret_key: "s3backupconfig.secret_key" 27 | bucket_name: "s3backupconfig.bucket_name" 28 | folder: "s3backupconfig.folder" 29 | endpoint: "s3backupconfig.endpoint" 30 | region: "s3backupconfig.region" 31 | 32 | cloud_provider: 33 | name: vsphere 34 | vsphereCloudProvider: 35 | disk: {} 36 | global: 37 | insecure-flag: true 38 | soap-roundtrip-count: 0 39 | network: {} 40 | virtual_center: 41 | cloud_provider.virtual_center: 42 | datacenters: cloud_provider.datacenters 43 | port: cloud_provider.port 44 | soap-roundtrip-count: 0 45 | user: cloud_provider.username 46 | password: cloud_provider.password 47 | workspace: 48 | datacenter: cloud_provider.datacenters 49 | default-datastore: cloud_provider.default-datastore 50 | folder: cloud_provider.folder 51 | server: cloud_provider.virtual_center 52 | 53 | dns: 54 | provider: coredns 55 | upstreamnameservers: 56 | - 1.1.1.1 57 | - 1.0.0.1 58 | 59 | kubernetes_version: v1.19.7-rancher1-1 60 | -------------------------------------------------------------------------------- /clusters/template/rancher-values.yaml: -------------------------------------------------------------------------------- 1 | antiAffinity: required 2 | auditLog: 3 | level: 2 4 | hostname: rancher.example.com 5 | ingress: 6 | tls: 7 | source: secret 8 | rancher_verison: 2.5.5 9 | rancher_chart: rancher-latest/rancher 10 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CWD=`pwd` 3 | timestamp() { 4 | date "+%Y-%m-%d %H:%M:%S" 5 | } 6 | techo() { 7 | echo "$(timestamp): $*" 8 | } 9 | decho() { 10 | if [[ ! -z $DEBUG ]] 11 | then 12 | techo "$*" 13 | fi 14 | } 15 | setup-ssh() { 16 | echo "Setting up SSH key..." 17 | if [[ -z $SSH_KEY ]] 18 | then 19 | echo "SSH Key is missing" 20 | exit 1 21 | fi 22 | mkdir /root/.ssh && echo "$SSH_KEY" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa 23 | } 24 | verify-files() { 25 | if [[ "$DEBUG" == "true" ]] 26 | then 27 | ls -lh $CWD"/clusters/"$Cluster 28 | fi 29 | if [[ ! -d "$CWD"/clusters/"$Cluster" ]] && [[ ! "$Cluster" == "all" ]] 30 | then 31 | echo "Cluster folder is missing" 32 | exit 2 33 | fi 34 | if [[ ! -f "$CWD"/clusters/"$Cluster"/cluster.yml ]] && [[ ! "$Cluster" == "all" ]] 35 | then 36 | echo "cluster.yml is missing" 37 | exit 3 38 | fi 39 | } 40 | pull-files-from-s3() { 41 | aws s3 sync --exclude="cluster.yml" --endpoint-url="$S3_ENDPOINT" s3://"$S3_BUCKET"/clusters/"$Cluster"/ "$CWD"/clusters/"$Cluster"/ 42 | } 43 | push-files-to-s3() { 44 | aws s3 sync --endpoint-url="$S3_ENDPOINT" "$CWD"/clusters/"$Cluster"/ s3://"$S3_BUCKET"/clusters/"$Cluster"/ 45 | } 46 | update-creds-in-cluster-yml() { 47 | cd "$CWD"/clusters/"$Cluster" 48 | if [[ -f ./creds ]] 49 | then 50 | techo "Found creds file, updating cluster.yml" 51 | while read line; 52 | do 53 | find=`echo $line | awk -F '=' '{print $1}'` 54 | replace=`echo $line | awk -F '=' '{print $2}'` 55 | techo "Find and replacing value for $find" 56 | cat cluster.yml | sed "s|${find}|${replace}|g" > cluster.tmp 57 | if [[ -z cluster.tmp ]] 58 | then 59 | echo "Problem" 60 | exit 5 61 | fi 62 | mv cluster.tmp cluster.yml 63 | done < ./creds 64 | techo "Updated creds in cluster.yml" 65 | else 66 | techo "No creds file, skipping" 67 | fi 68 | } 69 | rolling_reboot() { 70 | pull-files-from-s3 71 | cd "$CWD"/clusters/"$Cluster" 72 | export KUBECONFIG=./kube_config_cluster.yml 73 | for node in `kubectl get nodes -o name | awk -F'/' '{print $2}'` 74 | do 75 | echo "Node: $node" 76 | ipaddress=`kubectl get node $node -o jsonpath='{.metadata.annotations.rke\.cattle\.io/external-ip}'` 77 | if [[ -z $ipaddress ]] 78 | then 79 | ipaddress=`kubectl get node $node -o jsonpath='{.metadata.annotations.rke\.cattle\.io/internal-ip}'` 80 | fi 81 | echo "IpAddress: $ipaddress" 82 | status=`kubectl get nodes "$node" | tail -n1 | awk '{print $2}'` 83 | echo "Checking if node is ready..." 84 | if [[ "$status" == "Ready" ]] 85 | then 86 | echo "Cordoning node..." 87 | kubectl cordon "$node" 88 | ## Skipping drain to speed up rolling reboot 89 | #kubectl drain "$node" --ignore-daemonsets --delete-local-data --force --grace-period=60 90 | echo "Updating..." 91 | ssh -q -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null root@"$ipaddress" 'apt update -y && apt upgrade -y; reboot' 92 | echo "Rebooting..." 93 | sleep 30 94 | echo "Waiting for ping..." 95 | while ! ping -c 1 $ipaddress 96 | do 97 | echo "Waiting..." 98 | sleep 1 99 | done 100 | echo "Waiting for docker..." 101 | while ! ssh -q -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null root@"$ipaddress" "docker ps" 102 | do 103 | echo "Waiting..." 104 | sleep 1 105 | done 106 | echo "Waiting for node ready..." 107 | while ! kubectl get nodes "$node" | tail -n1 | awk '{print $2}' | grep "Ready" 108 | do 109 | echo "Waiting..." 110 | sleep 1 111 | done 112 | echo "Uncordoning node..." 113 | kubectl uncordon "$node" 114 | rke up 115 | else 116 | echo "Uncordoning node..." 117 | kubectl uncordon "$node" 118 | fi 119 | done 120 | push-files-to-s3 121 | } 122 | etcd_snapshot() { 123 | if [[ ! -z "$1" ]] 124 | then 125 | SnapshotName=$1 126 | else 127 | SnapshotName="builder-"`date "+%Y-%m-%d-%H-%M-%S"` 128 | fi 129 | pull-files-from-s3 $Cluster 130 | cd "$CWD"/clusters/"$Cluster" 131 | update-creds-in-cluster-yml 132 | techo "Taking etcd snapshot" 133 | techo "Snapshot Name: $SnapshotName" 134 | rke etcd snapshot-save --name "$SnapshotName" --config cluster.yml 135 | } 136 | cluster_up() { 137 | pull-files-from-s3 $Cluster 138 | cd "$CWD"/clusters/"$Cluster" 139 | update-creds-in-cluster-yml 140 | if [[ "$DEBUG" == "true" ]] 141 | then 142 | rke up --debug --config cluster.yml 143 | else 144 | rke up --config cluster.yml 145 | fi 146 | push-files-to-s3 $Cluster 147 | } 148 | cluster_delete() { 149 | pull-files-from-s3 $Cluster 150 | cd "$CWD"/clusters/"$Cluster" 151 | if [[ "$DEBUG" == "true" ]] 152 | then 153 | rke remove --debug --config cluster.yml 154 | else 155 | rke remove --config cluster.yml 156 | fi 157 | push-files-to-s3 $Cluster 158 | } 159 | install_cert-manager() { 160 | kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.crds.yaml 161 | kubectl create namespace cert-manager --dry-run=client -o yaml | kubectl apply -f - 162 | helm repo add jetstack https://charts.jetstack.io 163 | helm repo update 164 | helm upgrade --install \ 165 | cert-manager jetstack/cert-manager \ 166 | --namespace cert-manager \ 167 | --version v1.0.4 168 | } 169 | rancher_up() { 170 | cd "$CWD"/clusters/"$Cluster" 171 | techo "Checking for rancher-values.yaml" 172 | if [[ ! -f rancher-values.yaml ]] 173 | then 174 | techo "Missing rancher-values.yaml, canceling to Rancher Upgrade/Install" 175 | exit 8 176 | fi 177 | update-creds-in-cluster-yml 178 | techo "Taking per upgrade/install snapshot" 179 | SnapshotName="rancher-preupgrade-"`date "+%Y-%m-%d-%H-%M-%S"` 180 | etcd_snapshot $SnapshotName 181 | RC=$? 182 | if [ $RC -ne 0 ] 183 | then 184 | techo "etcd snapshot failed, canceling to Rancher Upgrade/Install" 185 | exit 6 186 | else 187 | techo "etcd snapshot was successful, processing to Rancher Upgrade/Install" 188 | fi 189 | techo "Verifing cluster access" 190 | export KUBECONFIG=kube_config_cluster.yml 191 | kubectl get nodes -o wide 192 | RC=$? 193 | if [ $RC -ne 0 ] 194 | then 195 | techo "Access failed, canceling to Rancher Upgrade/Install" 196 | exit 7 197 | fi 198 | techo "Creating cattle-system namespace" 199 | kubectl create namespace cattle-system --dry-run=client -o yaml | kubectl apply -f - 200 | techo "Setting up certs" 201 | if cat ./rancher-values.yaml | grep -A 3 'ingress' | grep 'source: secret' 202 | then 203 | techo "Certificates from Files" 204 | if [[ -f tls.crt ]] && [[ -f tls.key ]] 205 | then 206 | techo "Adding tls.crt and tls.key from s3" 207 | kubectl -n cattle-system create secret tls tls-rancher-ingress --cert=tls.crt --key=tls.key --dry-run=client -o yaml | kubectl apply -f - 208 | else 209 | techo "Missing tls.crt and tls.key, canceling to Rancher Upgrade/Install" 210 | exit 9 211 | fi 212 | elif cat ./rancher-values.yaml | grep -A 3 'ingress' | grep 'source: letsEncrypt' 213 | then 214 | techo "Let’s Encrypt configured, installing cert-manager" 215 | install_cert-manager 216 | else 217 | techo "Rancher Generated Certificates (Default) configured, installing cert-manager" 218 | install_cert-manager 219 | fi 220 | techo "Adding Rancher helm repos" 221 | RancherChartLong=`cat ./rancher-values.yaml | grep 'rancher_chart:' | awk '{print $2}'` 222 | RancherChartShort=`echo $RancherChartLong | awk -F '/' '{print $1}'` 223 | RancherChartUrlEnd=`echo $RancherChartShort | awk -F '-' '{print $2}' | awk -F '/' '{print $1}'` 224 | if [[ -z $RancherChart ]] 225 | then 226 | RancherChartShort="rancher-latest" 227 | RancherChartUrlEnd="latest" 228 | fi 229 | techo "RancherChartLong: $RancherChartLong" 230 | techo "RancherChartShort: $RancherChartShort" 231 | techo "RancherChartUrlEnd: $RancherChartUrlEnd" 232 | helm repo add "$RancherChartShort" https://releases.rancher.com/server-charts/"$RancherChartUrlEnd" 233 | techo "Fetching charts" 234 | helm repo update 235 | techo "Deploying Rancher" 236 | RancherVerison=`cat ./rancher-values.yaml | grep 'rancher_verison:' | awk '{print $2}'` 237 | if [[ -z $RancherVerison ]] 238 | then 239 | techo "Installing/Upgrading Rancher to latest" 240 | helm upgrade --install rancher "$RancherChartLong" --namespace cattle-system -f rancher-values.yaml 241 | else 242 | techo "Installing/Upgrading Rancher to $RancherVerison" 243 | helm upgrade --install rancher "$RancherChartLong" --namespace cattle-system -f rancher-values.yaml --version "$RancherVerison" 244 | fi 245 | techo "Waiting for Rancher to be rolled out" 246 | kubectl -n cattle-system rollout status deploy/rancher -w 247 | techo "Taking post upgrade/install snapshot" 248 | SnapshotName="rancher-postupgrade-"`date "+%Y-%m-%d-%H-%M-%S"` 249 | etcd_snapshot $SnapshotName 250 | RC=$? 251 | if [ $RC -ne 0 ] 252 | then 253 | techo "etcd snapshot failed" 254 | exit 10 255 | else 256 | techo "etcd snapshot was successful" 257 | fi 258 | } 259 | 260 | #### Starting Main 261 | if [[ -z $Action ]] 262 | then 263 | echo "Action must be set" 264 | exit 0 265 | fi 266 | setup-ssh 267 | verify-files 268 | 269 | if [[ "$Action" == "cluster_up" ]] 270 | then 271 | if [[ "$Cluster" == "all" ]] 272 | then 273 | for Cluster in `ls ./clusters` 274 | do 275 | if [[ ! "$Cluster" == "template" ]] 276 | then 277 | techo "Cluster: $Cluster" 278 | cluster_up 279 | else 280 | techo "Skipping template" 281 | fi 282 | done 283 | else 284 | cluster_up 285 | fi 286 | elif [[ "$Action" == "cluster_delete" ]] 287 | then 288 | cluster_delete 289 | elif [[ "$Action" == "rolling_reboot" ]] 290 | then 291 | rolling_reboot 292 | elif [[ "$Action" == "rancher_up" ]] 293 | then 294 | rancher_up 295 | else 296 | techo "Action: $Action" 297 | techo "Unknown Action" 298 | exit 254 299 | fi 300 | -------------------------------------------------------------------------------- /s3-template/creds: -------------------------------------------------------------------------------- 1 | s3backupconfig.access_key=AAA 2 | s3backupconfig.secret_key=BBBBBBB 3 | s3backupconfig.bucket_name=etcd-backups 4 | s3backupconfig.folder=cluster-name 5 | s3backupconfig.endpoint=s3.url.example.com 6 | s3backupconfig.region=us-west-1 7 | cloud_provider.virtual_center=vcenter.example.com 8 | cloud_provider.datacenters=/Datacenter-Example 9 | cloud_provider.port=443 10 | cloud_provider.username=serviceaccount 11 | cloud_provider.password=Passw0rd 12 | cloud_provider.default-datastore=/Datacenter-Example/datastore/datastore01 13 | cloud_provider.folder=cluster-name 14 | --------------------------------------------------------------------------------