├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── TODO.md ├── certs ├── README.md ├── admin-csr.json ├── ca-config.json ├── ca-csr.json ├── create-ca.sh ├── gen-certs.sh ├── k8s-worker0-csr.json ├── k8s-worker1-csr.json ├── k8s-worker2-csr.json ├── kube-controller-manager-csr.json ├── kube-proxy-csr.json ├── kube-scheduler-csr.json ├── kubernetes-csr.json ├── rm-certs.sh └── service-account-csr.json ├── compute.tf ├── configs └── gen-config.sh ├── configure-kubectl ├── deployment.yaml ├── main.tf ├── networking.tf ├── outputs.tf ├── scripts ├── add-ssh-keys.sh ├── create-rbac.sh ├── gen-admin-config.sh ├── gen-api-server-certs.sh ├── gen-client-certs.sh ├── gen-controller-manager-config.sh ├── gen-kubelet-config.sh ├── gen-proxy-config.sh ├── gen-scheduler-config.sh ├── install-controller.sh ├── install-worker.sh ├── start-controller.sh ├── start-etcd.sh └── start-worker.sh ├── terraform.tfvars └── variables.tf /.gitattributes: -------------------------------------------------------------------------------- 1 | *.jpeg filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,python,terraform,virtualenv 3 | 4 | ### Python ### 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # IPython 80 | profile_default/ 81 | ipython_config.py 82 | 83 | # pyenv 84 | .python-version 85 | 86 | # celery beat schedule file 87 | celerybeat-schedule 88 | 89 | # SageMath parsed files 90 | *.sage.py 91 | 92 | # Environments 93 | .env 94 | .venv 95 | env/ 96 | venv/ 97 | ENV/ 98 | env.bak/ 99 | venv.bak/ 100 | 101 | # Spyder project settings 102 | .spyderproject 103 | .spyproject 104 | 105 | # Rope project settings 106 | .ropeproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | .dmypy.json 114 | dmypy.json 115 | 116 | ### Python Patch ### 117 | .venv/ 118 | 119 | ### Python.VirtualEnv Stack ### 120 | # Virtualenv 121 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 122 | [Bb]in 123 | [Ii]nclude 124 | [Ll]ib 125 | [Ll]ib64 126 | [Ll]ocal 127 | # [Ss]cripts # Custom comment out. 128 | pyvenv.cfg 129 | pip-selfcheck.json 130 | 131 | ### Terraform ### 132 | # Local .terraform directories 133 | **/.terraform/* 134 | 135 | # .tfstate files 136 | *.tfstate 137 | *.tfstate.* 138 | 139 | # Crash log files 140 | crash.log 141 | 142 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 143 | # .tfvars files are managed as part of configuration and so should be included in 144 | # version control. 145 | # 146 | # example.tfvars 147 | terraform.tfvars 148 | 149 | # Ignore override files as they are usually used to override ressources locally 150 | override.tf 151 | override.tf.json 152 | *_override.tf 153 | *_override.tf.json 154 | 155 | ### Vim ### 156 | # Swap 157 | [._]*.s[a-v][a-z] 158 | [._]*.sw[a-p] 159 | [._]s[a-rt-v][a-z] 160 | [._]ss[a-gi-z] 161 | [._]sw[a-p] 162 | 163 | # Session 164 | Session.vim 165 | *.vim 166 | 167 | # Temporary 168 | .netrwhist 169 | *~ 170 | # Auto-generated tag files 171 | tags 172 | # Persistent undo 173 | [._]*.un~ 174 | 175 | ### VirtualEnv ### 176 | # Virtualenv 177 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 178 | 179 | 180 | # End of https://www.gitignore.io/api/vim,python,terraform,virtualenv 181 | 182 | *.csr 183 | *.key 184 | *.pem 185 | configs/encryption-config.yaml 186 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes The Hard Way With Terraform 2 | 3 | ## Table Of Contents 4 | 5 | [Project Status](#project-status) 6 | 7 | [Getting Started](#getting-started) 8 | 9 | [Contributing](#contributing) 10 | 11 | [Motivation](#motivation) 12 | 13 | ## Project Status 14 | 15 | **PRE-ALPHA**. Please don't use the project for any production workloads. Also, be sure to spin your cluster down if you aren't using it as it will cost you--or your company--money. 16 | 17 | ## Getting Started 18 | 19 | 1. If you don't already have one, sign up for a [Google Cloud Platform account](https://cloud.google.com/). 20 | 1. Download the `gcloud` [command-line tool](https://cloud.google.com/sdk/gcloud/). 21 | 1. Create a project in GCP named `k8s-the-hard-way-tf`. 22 | 1. On the command line, run: `gcloud init` to set up your account credentials/project details to point at the newly created project. 23 | 1. [Create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) (`IAM & admin` > `Service accounts` > `+ CREATE SERVICE ACCOUNT`) in the `k8s-the-hard-way-tf` project with `Owner` permissions, create/download the `*.json` credentials generated by the service account, and place them in a safe location on your local machine. 24 | 1. Set your Google Application credentials using: `export GOOGLE_APPLICATION_CREDENTIALS=`. 25 | 1. Go to the `certs` directory, and run the `create-ca.sh` script, followed by the `gen-certs.sh` script. This will create a self-signed CA, and create/sign all of the generated certs with that CA. 26 | 1. Go to the `configs` directory, and run the `gen-config.sh` script. This will create the encryption configuration for the new Kubernetes cluster. 27 | 1. Add a public `ssh` key to your project metadata (`Compute Engine` > `Metadata` > `SSH Keys`) using the format ` `. The virtual machines you spin up will inherit this public key and allow the `remote-exec` provisioners to work. 28 | 1. Fill in the variables in `variables.tf` that have `TODO` next to them. 29 | 1. **OPTIONAL**: Edit the GitHub usernames in `scripts/add-ssh-keys.sh` to give trusted colleagues/friends access to the machines in your Kubernetes cluster (see the [Motivation](#motivation) section for more details). 30 | 1. Run: `terraform init` to initialize Terraform. 31 | 1. Run: `terraform plan` to see the planned changes that Terraform will make. 32 | 1. Run: `terraform apply` to create your Kubernetes cluster. **NOTE**: The first `terraform apply` might fail because the `Compute Engine API has not been used in project` before. Follow the link provided in the error message (should be something like `https://console.developers.google.com/apis/api/compute.googleapis.com/overview?project=0123456789`) and click `ENABLE`. After the API is enabled, rerun `terraform apply`. 33 | 1. Configure `kubectl` to use the new cluster by running the `configure-kubectl` script. 34 | 1. Run `kubectl get cs` (or another related command) to test the cluster's component statuses. 35 | 36 | ## Contributing 37 | 38 | Make a PR and we'll go from there! 39 | 40 | ## Motivation 41 | 42 | This cluster was designed for the purpose of enabling effective study for the [Certified Kubernetes Administrator (CKA) exam](https://www.cncf.io/certification/cka/). The vision was to define a custom, from-scratch Kubernetes cluster in Terraform (using Kelsey Hightower's [kubernetes-the-hard-way tutorial](https://github.com/kelseyhightower/kubernetes-the-hard-way) as a reference point--thanks Kelsey), so that you could spin up a Kubernetes cluster with access to all of its internal components--including the Control Plane--on-demand, and destroy it if the cluster became unusable, or you just want to start over. This code enabled the majority of the Cloud Engineers at [Nebulaworks](https://nebulaworks.com/) to effectively study for, and get their CKA through hands-on practice in a CKA-exam-like environment (see below). 43 | 44 | How we would practice creating/using [Kubernetes objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) and completing [tasks](https://kubernetes.io/docs/tasks/) for the CKA: 45 | 1. Run `terraform apply` to create your Kubernetes cluster. 46 | 1. Scour the [Kubernetes reference docs](https://kubernetes.io/docs/tasks/) and try to create the objects/complete the tasks that you see. 47 | 48 | How we would practice for things breaking in the CKA: 49 | 1. Edit `scripts/add-ssh-keys.sh` to give trusted colleagues/friends access to the machines in your Kubernetes cluster. 50 | 1. Run `terraform apply` to create your Kubernetes cluster. 51 | 1. Have your trusted colleagues/friends `ssh` into your cluster and break things [you'll need to give them the username for your machines (run `whoami`), and the IP addresses of your machines (run `terraform output`)]. 52 | 1. Fix the things that are broken. 53 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | 1. Need to parameterize `networking.tf` based off of `worker_count` variables in `variables.tf` (currently hardcoded). 4 | 1. Need to lock down permissions for the service account that `terraform` uses (current `Owner` permissions are too broad). 5 | -------------------------------------------------------------------------------- /certs/README.md: -------------------------------------------------------------------------------- 1 | # Creating A Certificate Authority And Generating TLS Certificates 2 | 3 | ## Credits 4 | 5 | Code snippets in this repo taken from Kelsey Hightower's "Kubernetes The Hard Way" (found [here](https://github.com/kelseyhightower/kubernetes-the-hard-way)). 6 | 7 | Thank you Kelsey... 😭 8 | 9 | ![2 Legit To Quit...](https://media.giphy.com/media/SxiUFzAh5DgCk/giphy.gif) 10 | -------------------------------------------------------------------------------- /certs/admin-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "admin", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "system:masters", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/ca-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "8760h" 5 | }, 6 | "profiles": { 7 | "kubernetes": { 8 | "usages": ["signing", "key encipherment", "server auth", "client auth"], 9 | "expiry": "8760h" 10 | } 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /certs/ca-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "Kubernetes", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "Kubernetes", 12 | "OU": "CA", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/create-ca.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | check_prereqs () { 4 | declare -a req_progs=("cfssl" "cfssljson") 5 | 6 | for prog in "${req_progs[@]}" 7 | do 8 | if ! which "${prog}" > /dev/null 2>&1; then 9 | echo "${prog} not installed. Please install it then rerun this script." 10 | exit 1 11 | fi 12 | done 13 | } 14 | 15 | gen_ca_cert () { 16 | cfssl gencert -initca ca-csr.json | cfssljson -bare ca 17 | } 18 | 19 | check_prereqs 20 | gen_ca_cert 21 | -------------------------------------------------------------------------------- /certs/gen-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | check_prereqs () { 4 | declare -a req_progs=("cfssl" "cfssljson") 5 | 6 | for prog in "${req_progs[@]}" 7 | do 8 | if ! which "${prog}" > /dev/null 2>&1; then 9 | echo "${prog} not installed. Please install it then rerun this script." 10 | exit 1 11 | fi 12 | done 13 | } 14 | 15 | gen_admin_client_cert () { 16 | cfssl gencert \ 17 | -ca=ca.pem \ 18 | -ca-key=ca-key.pem \ 19 | -config=ca-config.json \ 20 | -profile=kubernetes \ 21 | admin-csr.json | cfssljson -bare admin 22 | } 23 | 24 | gen_controller_manager_cert () { 25 | cfssl gencert \ 26 | -ca=ca.pem \ 27 | -ca-key=ca-key.pem \ 28 | -config=ca-config.json \ 29 | -profile=kubernetes \ 30 | kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager 31 | } 32 | 33 | gen_kubernetes_api_cert () { 34 | cfssl gencert \ 35 | -ca=ca.pem \ 36 | -ca-key=ca-key.pem \ 37 | -config=ca-config.json \ 38 | -hostname=10.32.0.1,10.240.0.10,10.240.0.11,10.240.0.12,"${KUBERNETES_PUBLIC_ADDRESS}",127.0.0.1,kubernetes.default \ 39 | -profile=kubernetes \ 40 | kubernetes-csr.json | cfssljson -bare kubernetes 41 | } 42 | 43 | gen_kube_proxy_client_cert () { 44 | cfssl gencert \ 45 | -ca=ca.pem \ 46 | -ca-key=ca-key.pem \ 47 | -config=ca-config.json \ 48 | -profile=kubernetes \ 49 | kube-proxy-csr.json | cfssljson -bare kube-proxy 50 | } 51 | 52 | gen_scheduler_client_cert () { 53 | cfssl gencert \ 54 | -ca=ca.pem \ 55 | -ca-key=ca-key.pem \ 56 | -config=ca-config.json \ 57 | -profile=kubernetes \ 58 | kube-scheduler-csr.json | cfssljson -bare kube-scheduler 59 | } 60 | 61 | gen_service_account_kp () { 62 | cfssl gencert \ 63 | -ca=ca.pem \ 64 | -ca-key=ca-key.pem \ 65 | -config=ca-config.json \ 66 | -profile=kubernetes \ 67 | service-account-csr.json | cfssljson -bare service-account 68 | } 69 | 70 | check_prereqs 71 | gen_admin_client_cert 72 | gen_controller_manager_cert 73 | gen_kube_proxy_client_cert 74 | gen_scheduler_client_cert 75 | gen_kubernetes_api_cert 76 | gen_service_account_kp 77 | -------------------------------------------------------------------------------- /certs/k8s-worker0-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:node:k8s-worker0", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "system:nodes", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/k8s-worker1-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:node:k8s-worker1", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "system:nodes", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/k8s-worker2-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:node:k8s-worker2", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "system:nodes", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/kube-controller-manager-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-controller-manager", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "🤘", 10 | "L": "SLIM JIM", 11 | "O": "system:kube-controller-manager", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/kube-proxy-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-proxy", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "system:node-proxier", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/kube-scheduler-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-scheduler", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "system:kube-scheduler", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/kubernetes-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "kubernetes", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "Kubernetes", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certs/rm-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo 'Removing *.pem files...' 4 | rm *.pem 5 | 6 | echo 'Done.' 7 | -------------------------------------------------------------------------------- /certs/service-account-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "service-accounts", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "Randyville", 10 | "L": "SLIM JIM", 11 | "O": "Kubernetes", 12 | "OU": "OHH YYYEEEEAAAAAHHHHHH!!!!!!!!", 13 | "ST": "Macho Land" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /compute.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_instance" "k8s_controller" { 2 | boot_disk { 3 | auto_delete = true 4 | 5 | initialize_params { 6 | image = "${var.controller_image}" 7 | size = "${var.controller_size}" 8 | } 9 | } 10 | 11 | can_ip_forward = true 12 | count = "${var.controller_count}" 13 | machine_type = "${var.controller_type}" 14 | name = "k8s-controller${count.index}" 15 | 16 | network_interface { 17 | access_config = {} 18 | subnetwork = "${google_compute_subnetwork.k8s_subnet.name}" 19 | } 20 | 21 | metadata { 22 | creator = "${var.user}" 23 | } 24 | 25 | provisioner "file" { 26 | connection { 27 | private_key = "${file(var.ssh_path)}" 28 | user = "${var.user}" 29 | type = "ssh" 30 | } 31 | 32 | destination = "add-ssh-keys.sh" 33 | source = "${var.scripts_path}/add-ssh-keys.sh" 34 | } 35 | 36 | provisioner "file" { 37 | connection { 38 | private_key = "${file(var.ssh_path)}" 39 | user = "${var.user}" 40 | type = "ssh" 41 | } 42 | 43 | destination = "admin.pem" 44 | source = "${var.certs_path}/admin.pem" 45 | } 46 | 47 | provisioner "file" { 48 | connection { 49 | private_key = "${file(var.ssh_path)}" 50 | user = "${var.user}" 51 | type = "ssh" 52 | } 53 | 54 | destination = "admin-key.pem" 55 | source = "${var.certs_path}/admin-key.pem" 56 | } 57 | 58 | provisioner "file" { 59 | connection { 60 | private_key = "${file(var.ssh_path)}" 61 | user = "${var.user}" 62 | type = "ssh" 63 | } 64 | 65 | destination = "ca.pem" 66 | source = "${var.certs_path}/ca.pem" 67 | } 68 | 69 | provisioner "file" { 70 | connection { 71 | private_key = "${file(var.ssh_path)}" 72 | user = "${var.user}" 73 | type = "ssh" 74 | } 75 | 76 | destination = "ca-config.json" 77 | source = "${var.certs_path}/ca-config.json" 78 | } 79 | 80 | provisioner "file" { 81 | connection { 82 | private_key = "${file(var.ssh_path)}" 83 | user = "${var.user}" 84 | type = "ssh" 85 | } 86 | 87 | destination = "ca-key.pem" 88 | source = "${var.certs_path}/ca-key.pem" 89 | } 90 | 91 | provisioner "file" { 92 | connection { 93 | private_key = "${file(var.ssh_path)}" 94 | user = "${var.user}" 95 | type = "ssh" 96 | } 97 | 98 | destination = "create-rbac.sh" 99 | source = "${var.scripts_path}/create-rbac.sh" 100 | } 101 | 102 | provisioner "file" { 103 | connection { 104 | private_key = "${file(var.ssh_path)}" 105 | user = "${var.user}" 106 | type = "ssh" 107 | } 108 | 109 | destination = "encryption-config.yaml" 110 | source = "${var.configs_path}/encryption-config.yaml" 111 | } 112 | 113 | provisioner "file" { 114 | connection { 115 | private_key = "${file(var.ssh_path)}" 116 | user = "${var.user}" 117 | type = "ssh" 118 | } 119 | 120 | destination = "gen-admin-config.sh" 121 | source = "${var.scripts_path}/gen-admin-config.sh" 122 | } 123 | 124 | provisioner "file" { 125 | connection { 126 | private_key = "${file(var.ssh_path)}" 127 | user = "${var.user}" 128 | type = "ssh" 129 | } 130 | 131 | destination = "gen-api-server-certs.sh" 132 | source = "${var.scripts_path}/gen-api-server-certs.sh" 133 | } 134 | 135 | provisioner "file" { 136 | connection { 137 | private_key = "${file(var.ssh_path)}" 138 | user = "${var.user}" 139 | type = "ssh" 140 | } 141 | 142 | destination = "gen-controller-manager-config.sh" 143 | source = "${var.scripts_path}/gen-controller-manager-config.sh" 144 | } 145 | 146 | provisioner "file" { 147 | connection { 148 | private_key = "${file(var.ssh_path)}" 149 | user = "${var.user}" 150 | type = "ssh" 151 | } 152 | 153 | destination = "gen-scheduler-config.sh" 154 | source = "${var.scripts_path}/gen-scheduler-config.sh" 155 | } 156 | 157 | provisioner "file" { 158 | connection { 159 | private_key = "${file(var.ssh_path)}" 160 | user = "${var.user}" 161 | type = "ssh" 162 | } 163 | 164 | destination = "install-controller.sh" 165 | source = "${var.scripts_path}/install-controller.sh" 166 | } 167 | 168 | provisioner "file" { 169 | connection { 170 | private_key = "${file(var.ssh_path)}" 171 | user = "${var.user}" 172 | type = "ssh" 173 | } 174 | 175 | destination = "kubernetes-csr.json" 176 | source = "${var.certs_path}/kubernetes-csr.json" 177 | } 178 | 179 | provisioner "file" { 180 | connection { 181 | private_key = "${file(var.ssh_path)}" 182 | user = "${var.user}" 183 | type = "ssh" 184 | } 185 | 186 | destination = "kube-controller-manager.pem" 187 | source = "${var.certs_path}/kube-controller-manager.pem" 188 | } 189 | 190 | provisioner "file" { 191 | connection { 192 | private_key = "${file(var.ssh_path)}" 193 | user = "${var.user}" 194 | type = "ssh" 195 | } 196 | 197 | destination = "kube-controller-manager-key.pem" 198 | source = "${var.certs_path}/kube-controller-manager-key.pem" 199 | } 200 | 201 | provisioner "file" { 202 | connection { 203 | private_key = "${file(var.ssh_path)}" 204 | user = "${var.user}" 205 | type = "ssh" 206 | } 207 | 208 | destination = "kube-scheduler.pem" 209 | source = "${var.certs_path}/kube-scheduler.pem" 210 | } 211 | 212 | provisioner "file" { 213 | connection { 214 | private_key = "${file(var.ssh_path)}" 215 | user = "${var.user}" 216 | type = "ssh" 217 | } 218 | 219 | destination = "kube-scheduler-key.pem" 220 | source = "${var.certs_path}/kube-scheduler-key.pem" 221 | } 222 | 223 | provisioner "file" { 224 | connection { 225 | private_key = "${file(var.ssh_path)}" 226 | user = "${var.user}" 227 | type = "ssh" 228 | } 229 | 230 | destination = "service-account.pem" 231 | source = "${var.certs_path}/service-account.pem" 232 | } 233 | 234 | provisioner "file" { 235 | connection { 236 | private_key = "${file(var.ssh_path)}" 237 | user = "${var.user}" 238 | type = "ssh" 239 | } 240 | 241 | destination = "service-account-key.pem" 242 | source = "${var.certs_path}/service-account-key.pem" 243 | } 244 | 245 | provisioner "file" { 246 | connection { 247 | private_key = "${file(var.ssh_path)}" 248 | user = "${var.user}" 249 | type = "ssh" 250 | } 251 | 252 | destination = "start-controller.sh" 253 | source = "${var.scripts_path}/start-controller.sh" 254 | } 255 | 256 | provisioner "file" { 257 | connection { 258 | private_key = "${file(var.ssh_path)}" 259 | user = "${var.user}" 260 | type = "ssh" 261 | } 262 | 263 | destination = "start-etcd.sh" 264 | source = "${var.scripts_path}/start-etcd.sh" 265 | } 266 | 267 | provisioner "remote-exec" { 268 | connection { 269 | private_key = "${file(var.ssh_path)}" 270 | user = "${var.user}" 271 | type = "ssh" 272 | } 273 | 274 | inline = [ 275 | "sleep 30", 276 | "sudo chmod +x add-ssh-keys.sh create-rbac.sh gen-admin-config.sh gen-api-server-certs.sh gen-controller-manager-config.sh gen-scheduler-config.sh install-controller.sh start-controller.sh start-etcd.sh", 277 | "./add-ssh-keys.sh", 278 | "./install-controller.sh", 279 | "./gen-api-server-certs.sh ${var.project} ${google_compute_address.k8s_staticip.address}", 280 | "./gen-controller-manager-config.sh", 281 | "./gen-scheduler-config.sh", 282 | "./gen-admin-config.sh", 283 | "./start-etcd.sh ${var.project}", 284 | "./start-controller.sh ${var.project}", 285 | "./create-rbac.sh", 286 | ] 287 | } 288 | 289 | service_account { 290 | scopes = ["compute-rw", "storage-ro", "service-management", "service-control", "logging-write", "monitoring"] 291 | } 292 | 293 | tags = ["controller"] 294 | zone = "${var.region}-${lookup(var.zone_map, count.index)}" 295 | } 296 | 297 | resource "google_compute_instance" "k8s_worker" { 298 | boot_disk { 299 | auto_delete = true 300 | 301 | initialize_params { 302 | image = "${var.worker_image}" 303 | size = "${var.worker_size}" 304 | } 305 | } 306 | 307 | can_ip_forward = true 308 | count = "${var.worker_count}" 309 | machine_type = "${var.worker_type}" 310 | name = "k8s-worker${count.index}" 311 | 312 | network_interface { 313 | access_config = {} 314 | subnetwork = "${google_compute_subnetwork.k8s_subnet.name}" 315 | } 316 | 317 | metadata { 318 | creator = "${var.user}" 319 | pod-cidr = "10.200.${count.index}.0/24" 320 | } 321 | 322 | provisioner "file" { 323 | connection { 324 | private_key = "${file(var.ssh_path)}" 325 | user = "${var.user}" 326 | type = "ssh" 327 | } 328 | 329 | destination = "add-ssh-keys.sh" 330 | source = "${var.scripts_path}/add-ssh-keys.sh" 331 | } 332 | 333 | provisioner "file" { 334 | connection { 335 | private_key = "${file(var.ssh_path)}" 336 | user = "${var.user}" 337 | type = "ssh" 338 | } 339 | 340 | destination = "ca.pem" 341 | source = "${var.certs_path}/ca.pem" 342 | } 343 | 344 | provisioner "file" { 345 | connection { 346 | private_key = "${file(var.ssh_path)}" 347 | user = "${var.user}" 348 | type = "ssh" 349 | } 350 | 351 | destination = "ca-key.pem" 352 | source = "${var.certs_path}/ca-key.pem" 353 | } 354 | 355 | provisioner "file" { 356 | connection { 357 | private_key = "${file(var.ssh_path)}" 358 | user = "${var.user}" 359 | type = "ssh" 360 | } 361 | 362 | destination = "ca-config.json" 363 | source = "${var.certs_path}/ca-config.json" 364 | } 365 | 366 | provisioner "file" { 367 | connection { 368 | private_key = "${file(var.ssh_path)}" 369 | user = "${var.user}" 370 | type = "ssh" 371 | } 372 | 373 | destination = "gen-client-certs.sh" 374 | source = "${var.scripts_path}/gen-client-certs.sh" 375 | } 376 | 377 | provisioner "file" { 378 | connection { 379 | private_key = "${file(var.ssh_path)}" 380 | user = "${var.user}" 381 | type = "ssh" 382 | } 383 | 384 | destination = "gen-kubelet-config.sh" 385 | source = "${var.scripts_path}/gen-kubelet-config.sh" 386 | } 387 | 388 | provisioner "file" { 389 | connection { 390 | private_key = "${file(var.ssh_path)}" 391 | user = "${var.user}" 392 | type = "ssh" 393 | } 394 | 395 | destination = "gen-proxy-config.sh" 396 | source = "${var.scripts_path}/gen-proxy-config.sh" 397 | } 398 | 399 | provisioner "file" { 400 | connection { 401 | private_key = "${file(var.ssh_path)}" 402 | user = "${var.user}" 403 | type = "ssh" 404 | } 405 | 406 | destination = "kube-proxy.pem" 407 | source = "${var.certs_path}/kube-proxy.pem" 408 | } 409 | 410 | provisioner "file" { 411 | connection { 412 | private_key = "${file(var.ssh_path)}" 413 | user = "${var.user}" 414 | type = "ssh" 415 | } 416 | 417 | destination = "kube-proxy-key.pem" 418 | source = "${var.certs_path}/kube-proxy-key.pem" 419 | } 420 | 421 | provisioner "file" { 422 | connection { 423 | private_key = "${file(var.ssh_path)}" 424 | user = "${var.user}" 425 | type = "ssh" 426 | } 427 | 428 | destination = "install-worker.sh" 429 | source = "${var.scripts_path}/install-worker.sh" 430 | } 431 | 432 | provisioner "file" { 433 | connection { 434 | private_key = "${file(var.ssh_path)}" 435 | user = "${var.user}" 436 | type = "ssh" 437 | } 438 | 439 | destination = "k8s-worker${count.index}-csr.json" 440 | source = "${var.certs_path}/k8s-worker${count.index}-csr.json" 441 | } 442 | 443 | service_account { 444 | scopes = ["compute-rw", "storage-ro", "service-management", "service-control", "logging-write", "monitoring"] 445 | } 446 | 447 | provisioner "file" { 448 | connection { 449 | private_key = "${file(var.ssh_path)}" 450 | user = "${var.user}" 451 | type = "ssh" 452 | } 453 | 454 | destination = "start-worker.sh" 455 | source = "${var.scripts_path}/start-worker.sh" 456 | } 457 | 458 | provisioner "remote-exec" { 459 | connection { 460 | private_key = "${file(var.ssh_path)}" 461 | user = "${var.user}" 462 | type = "ssh" 463 | } 464 | 465 | inline = [ 466 | "sleep 30", 467 | "sudo chmod +x add-ssh-keys.sh gen-client-certs.sh gen-kubelet-config.sh gen-proxy-config.sh install-worker.sh start-worker.sh", 468 | "./add-ssh-keys.sh", 469 | "./install-worker.sh", 470 | "./gen-client-certs.sh", 471 | "./gen-kubelet-config.sh ${google_compute_address.k8s_staticip.address}", 472 | "./gen-proxy-config.sh ${google_compute_address.k8s_staticip.address}", 473 | "./start-worker.sh", 474 | ] 475 | } 476 | 477 | tags = ["worker"] 478 | zone = "${var.region}-${lookup(var.zone_map, count.index)}" 479 | } 480 | -------------------------------------------------------------------------------- /configs/gen-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64) 4 | 5 | cat > encryption-config.yaml <> ~/.ssh/authorized_keys 18 | curl $CURL_FLAGS "https://github.com/${account}.keys" >> ~/.ssh/authorized_keys 19 | done 20 | } 21 | 22 | add_github_pub_keys 23 | -------------------------------------------------------------------------------- /scripts/create-rbac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | cat > rbac-cr.yaml < rbac-crb.yaml <> cloudinit.log 50 | -------------------------------------------------------------------------------- /scripts/gen-admin-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | kubectl config set-cluster k8s-the-hard-way \ 7 | --certificate-authority=ca.pem \ 8 | --embed-certs=true \ 9 | --server=https://127.0.0.1:6443 \ 10 | --kubeconfig=admin.kubeconfig 11 | 12 | kubectl config set-credentials admin \ 13 | --client-certificate=admin.pem \ 14 | --client-key=admin-key.pem \ 15 | --embed-certs=true \ 16 | --kubeconfig=admin.kubeconfig 17 | 18 | kubectl config set-context default \ 19 | --cluster=k8s-the-hard-way \ 20 | --user=admin \ 21 | --kubeconfig=admin.kubeconfig 22 | 23 | kubectl config use-context default --kubeconfig=admin.kubeconfig 24 | } >> cloudinit.log 25 | -------------------------------------------------------------------------------- /scripts/gen-api-server-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | PROJ_NAME=$1 7 | CONTROLLER0_IP=$(dig +short k8s-controller0.us-west1-a.c."$PROJ_NAME".internal) 8 | CONTROLLER1_IP=$(dig +short k8s-controller1.us-west1-b.c."$PROJ_NAME".internal) 9 | CONTROLLER2_IP=$(dig +short k8s-controller2.us-west1-c.c."$PROJ_NAME".internal) 10 | KUBERNETES_PUBLIC_ADDRESS=$2 11 | 12 | cfssl gencert \ 13 | -ca=ca.pem \ 14 | -ca-key=ca-key.pem \ 15 | -config=ca-config.json \ 16 | -hostname=10.32.0.1,"${CONTROLLER0_IP}","${CONTROLLER1_IP}","${CONTROLLER2_IP}","${KUBERNETES_PUBLIC_ADDRESS}",127.0.0.1,kubernetes.default \ 17 | -profile=kubernetes \ 18 | kubernetes-csr.json | cfssljson -bare kubernetes 19 | } >> cloudinit.log 20 | -------------------------------------------------------------------------------- /scripts/gen-client-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | EXTERNAL_IP=$(curl -s "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" -H "Metadata-Flavor: Google") 7 | echo "$EXTERNAL_IP" 8 | NODE_NAME=$(curl -s "http://metadata/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google" | cut -d. -f1) 9 | echo "$NODE_NAME" 10 | INTERNAL_IP=$(curl -s "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip" -H "Metadata-Flavor: Google") 11 | echo "$INTERNAL_IP" 12 | 13 | cfssl gencert \ 14 | -ca=ca.pem \ 15 | -ca-key=ca-key.pem \ 16 | -config=ca-config.json \ 17 | -hostname="${NODE_NAME}","${EXTERNAL_IP}","${INTERNAL_IP}" \ 18 | -profile=kubernetes \ 19 | "${NODE_NAME}"-csr.json | cfssljson -bare "${NODE_NAME}" 20 | } >> cloudinit.log 21 | -------------------------------------------------------------------------------- /scripts/gen-controller-manager-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | kubectl config set-cluster k8s-the-hard-way \ 7 | --certificate-authority=ca.pem \ 8 | --embed-certs=true \ 9 | --server=https://127.0.0.1:6443 \ 10 | --kubeconfig=kube-controller-manager.kubeconfig 11 | 12 | kubectl config set-credentials system:kube-controller-manager \ 13 | --client-certificate=kube-controller-manager.pem \ 14 | --client-key=kube-controller-manager-key.pem \ 15 | --embed-certs=true \ 16 | --kubeconfig=kube-controller-manager.kubeconfig 17 | 18 | kubectl config set-context default \ 19 | --cluster=k8s-the-hard-way \ 20 | --user=system:kube-controller-manager \ 21 | --kubeconfig=kube-controller-manager.kubeconfig 22 | 23 | kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig 24 | } >> cloudinit.log 25 | -------------------------------------------------------------------------------- /scripts/gen-kubelet-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | KUBERNETES_PUBLIC_ADDRESS=$1 # Static IP address provisioned in networking.tf 7 | NODE_NAME=$(curl -s "http://metadata/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google" | cut -d. -f1) 8 | 9 | echo "$KUBERNETES_PUBLIC_ADDRESS" 10 | echo "$NODE_NAME" 11 | 12 | kubectl config set-cluster k8s-the-hard-way \ 13 | --certificate-authority=ca.pem \ 14 | --embed-certs=true \ 15 | --server=https://"${KUBERNETES_PUBLIC_ADDRESS}":6443 \ 16 | --kubeconfig="${NODE_NAME}".kubeconfig 17 | 18 | kubectl config set-credentials system:node:"${NODE_NAME}" \ 19 | --client-certificate="${NODE_NAME}".pem \ 20 | --client-key="${NODE_NAME}"-key.pem \ 21 | --embed-certs=true \ 22 | --kubeconfig="${NODE_NAME}".kubeconfig 23 | 24 | kubectl config set-context default \ 25 | --cluster=k8s-the-hard-way \ 26 | --user=system:node:"${NODE_NAME}" \ 27 | --kubeconfig="${NODE_NAME}".kubeconfig 28 | 29 | kubectl config use-context default --kubeconfig="${NODE_NAME}".kubeconfig 30 | } >> cloudinit.log 31 | -------------------------------------------------------------------------------- /scripts/gen-proxy-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | KUBERNETES_PUBLIC_ADDRESS=$1 # Static IP address provisioned in networking.tf 7 | 8 | kubectl config set-cluster k8s-the-hard-way \ 9 | --certificate-authority=ca.pem \ 10 | --embed-certs=true \ 11 | --server=https://"${KUBERNETES_PUBLIC_ADDRESS}":6443 \ 12 | --kubeconfig=kube-proxy.kubeconfig 13 | 14 | kubectl config set-credentials system:kube-proxy \ 15 | --client-certificate=kube-proxy.pem \ 16 | --client-key=kube-proxy-key.pem \ 17 | --embed-certs=true \ 18 | --kubeconfig=kube-proxy.kubeconfig 19 | 20 | kubectl config set-context default \ 21 | --cluster=k8s-the-hard-way \ 22 | --user=system:kube-proxy \ 23 | --kubeconfig=kube-proxy.kubeconfig 24 | 25 | kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig 26 | } >> cloudinit.log 27 | -------------------------------------------------------------------------------- /scripts/gen-scheduler-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | kubectl config set-cluster k8s-the-hard-way \ 7 | --certificate-authority=ca.pem \ 8 | --embed-certs=true \ 9 | --server=https://127.0.0.1:6443 \ 10 | --kubeconfig=kube-scheduler.kubeconfig 11 | 12 | kubectl config set-credentials system:kube-scheduler \ 13 | --client-certificate=kube-scheduler.pem \ 14 | --client-key=kube-scheduler-key.pem \ 15 | --embed-certs=true \ 16 | --kubeconfig=kube-scheduler.kubeconfig 17 | 18 | kubectl config set-context default \ 19 | --cluster=k8s-the-hard-way \ 20 | --user=system:kube-scheduler \ 21 | --kubeconfig=kube-scheduler.kubeconfig 22 | 23 | kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig 24 | } >> cloudinit.log 25 | -------------------------------------------------------------------------------- /scripts/install-controller.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | # Install cfssl and cfssljson 7 | wget -q --https-only --timestamping \ 8 | https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ 9 | https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 10 | 11 | chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 12 | sudo mv cfssl_linux-amd64 /usr/local/bin/cfssl 13 | sudo mv cfssljson_linux-amd64 /usr/local/bin/cfssljson 14 | 15 | # Install kubectl 16 | sudo apt-get update && sudo apt-get install -y apt-transport-https 17 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - 18 | sudo touch /etc/apt/sources.list.d/kubernetes.list 19 | echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list 20 | sudo apt-get update 21 | sudo apt-get install -y kubectl 22 | 23 | # Install/prepare etcd 24 | wget -q --show-progress --https-only --timestamping \ 25 | "https://github.com/coreos/etcd/releases/download/v3.3.5/etcd-v3.3.5-linux-amd64.tar.gz" 26 | tar -xvf etcd-v3.3.5-linux-amd64.tar.gz 27 | sudo mv etcd-v3.3.5-linux-amd64/etcd* /usr/local/bin/ 28 | 29 | # Provision Kubernetes Control Plane 30 | sudo mkdir -p /etc/kubernetes/config 31 | wget -q --show-progress --https-only --timestamping \ 32 | "https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kube-apiserver" \ 33 | "https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kube-controller-manager" \ 34 | "https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kube-scheduler" \ 35 | "https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubectl" 36 | chmod +x kube-apiserver kube-controller-manager kube-scheduler kubectl 37 | sudo mv kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/ 38 | } >> cloudinit.log 39 | -------------------------------------------------------------------------------- /scripts/install-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | # Install cfssl and cfssljson 7 | wget -q --https-only --timestamping \ 8 | https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ 9 | https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 10 | 11 | chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 12 | sudo mv cfssl_linux-amd64 /usr/local/bin/cfssl 13 | sudo mv cfssljson_linux-amd64 /usr/local/bin/cfssljson 14 | 15 | # Install kubectl 16 | sudo apt-get update && sudo apt-get install -y apt-transport-https 17 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - 18 | sudo touch /etc/apt/sources.list.d/kubernetes.list 19 | echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list 20 | sudo apt-get update 21 | sudo apt-get install -y kubectl 22 | 23 | # Install OS Dependencies 24 | sudo apt-get update 25 | sudo apt-get -y install socat conntrack ipset 26 | 27 | 28 | # Install/Configure Worker Dependencies 29 | wget -q --show-progress --https-only --timestamping \ 30 | https://github.com/kubernetes-incubator/cri-tools/releases/download/v1.0.0-beta.0/crictl-v1.0.0-beta.0-linux-amd64.tar.gz \ 31 | https://storage.googleapis.com/kubernetes-the-hard-way/runsc \ 32 | https://github.com/opencontainers/runc/releases/download/v1.0.0-rc5/runc.amd64 \ 33 | https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz \ 34 | https://github.com/containerd/containerd/releases/download/v1.1.0/containerd-1.1.0.linux-amd64.tar.gz \ 35 | https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubectl \ 36 | https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kube-proxy \ 37 | https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubelet 38 | 39 | sudo mkdir -p \ 40 | /etc/cni/net.d \ 41 | /opt/cni/bin \ 42 | /var/lib/kubelet \ 43 | /var/lib/kube-proxy \ 44 | /var/lib/kubernetes \ 45 | /var/run/kubernetes 46 | 47 | chmod +x kubectl kube-proxy kubelet runc.amd64 runsc 48 | sudo mv runc.amd64 runc 49 | sudo mv kubectl kube-proxy kubelet runc runsc /usr/local/bin/ 50 | sudo tar -xvf crictl-v1.0.0-beta.0-linux-amd64.tar.gz -C /usr/local/bin/ 51 | sudo tar -xvf cni-plugins-amd64-v0.6.0.tgz -C /opt/cni/bin/ 52 | sudo tar -xvf containerd-1.1.0.linux-amd64.tar.gz -C / 53 | } >> cloudinit.log 54 | -------------------------------------------------------------------------------- /scripts/start-controller.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | PROJ_NAME=$1 7 | CONTROLLER0_IP=$(dig +short k8s-controller0.us-west1-a.c."$PROJ_NAME".internal) 8 | CONTROLLER1_IP=$(dig +short k8s-controller1.us-west1-b.c."$PROJ_NAME".internal) 9 | CONTROLLER2_IP=$(dig +short k8s-controller2.us-west1-c.c."$PROJ_NAME".internal) 10 | INTERNAL_IP=$(curl -s "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip" -H "Metadata-Flavor: Google") 11 | 12 | # Configure API Server 13 | sudo mkdir -p /var/lib/kubernetes/ 14 | 15 | sudo mv ca.pem ca-key.pem kubernetes-key.pem kubernetes.pem \ 16 | service-account-key.pem service-account.pem \ 17 | encryption-config.yaml /var/lib/kubernetes/ 18 | 19 | cat < kubernetes.default.svc.cluster.local <> cloudinit.log 155 | -------------------------------------------------------------------------------- /scripts/start-etcd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | PROJ_NAME=$1 7 | CONTROLLER0_IP=$(dig +short k8s-controller0.us-west1-a.c."$PROJ_NAME".internal) 8 | CONTROLLER1_IP=$(dig +short k8s-controller1.us-west1-b.c."$PROJ_NAME".internal) 9 | CONTROLLER2_IP=$(dig +short k8s-controller2.us-west1-c.c."$PROJ_NAME".internal) 10 | ETCD_NAME=$(hostname -s) 11 | INTERNAL_IP=$(curl -s "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip" -H "Metadata-Flavor: Google") 12 | 13 | sudo mkdir -p /etc/etcd /var/lib/etcd 14 | sudo cp ca.pem kubernetes-key.pem kubernetes.pem /etc/etcd/ 15 | 16 | cat <> cloudinit.log 58 | -------------------------------------------------------------------------------- /scripts/start-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | { 4 | echo && echo "$0: " && echo 5 | 6 | # Configure CNI Networking 7 | HOSTNAME=$(hostname -s) 8 | POD_CIDR=$(curl -s -H "Metadata-Flavor: Google" \ 9 | http://metadata.google.internal/computeMetadata/v1/instance/attributes/pod-cidr) 10 | 11 | cat <> cloudinit.log 157 | -------------------------------------------------------------------------------- /terraform.tfvars: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "certs_path" { 2 | default = "./certs" 3 | } 4 | 5 | variable "configs_path" { 6 | default = "./configs" 7 | } 8 | 9 | variable "controller_count" { 10 | default = 3 11 | } 12 | 13 | variable "controller_image" { 14 | default = "ubuntu-os-cloud/ubuntu-1804-lts" 15 | } 16 | 17 | variable "controller_size" { 18 | default = 100 19 | } 20 | 21 | variable "controller_type" { 22 | default = "n1-standard-1" 23 | } 24 | 25 | variable "project" { 26 | default = "k8s-the-hard-way-tf" # TODO: Might need to append to if GCP adds a number at the end. 27 | } 28 | 29 | variable "region" { 30 | default = "us-west1" 31 | } 32 | 33 | variable "scripts_path" { 34 | default = "./scripts" 35 | } 36 | 37 | variable "ssh_path" { 38 | default = "" # TODO: Path to private key that matches the public key added to the project metadata. 39 | } 40 | 41 | variable "user" { 42 | default = "" # TODO: Username on local system (run `whoami` to get this value). 43 | } 44 | 45 | variable "worker_count" { 46 | default = 3 47 | } 48 | 49 | variable "worker_image" { 50 | default = "ubuntu-os-cloud/ubuntu-1804-lts" 51 | } 52 | 53 | variable "worker_size" { 54 | default = 100 55 | } 56 | 57 | variable "worker_type" { 58 | default = "n1-standard-1" 59 | } 60 | 61 | variable "zone" { 62 | default = "us-west1-a" 63 | } 64 | 65 | variable "zone_map" { 66 | default = { 67 | "0" = "a" 68 | "1" = "b" 69 | "2" = "c" 70 | } 71 | } 72 | --------------------------------------------------------------------------------