├── .circleci └── config.yml ├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md ├── labels.yml ├── release-drafter.yml └── workflows │ ├── label-sync.yml │ └── release-drafter.yml ├── .gitignore ├── .terraform.lock.hcl ├── LICENSE ├── Makefile ├── README.md ├── config └── sshd_config ├── main.tf ├── modules ├── instances │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── scripts │ │ ├── end.sh │ │ ├── kubeadm-init.sh │ │ ├── kubeadm-install.sh │ │ ├── linode-addons.sh │ │ ├── linode-network.sh │ │ ├── monitoring-install.sh │ │ └── start.sh │ ├── variables.tf │ └── versions.tf ├── masters │ ├── README.md │ ├── main.tf │ ├── manifests │ │ ├── .gitignore │ │ ├── calico.yaml │ │ ├── dashboard-rbac.yaml │ │ ├── dashboard.yaml │ │ ├── external-dns.yaml │ │ ├── linode-token.yaml │ │ └── metrics-server.yaml │ ├── outputs.tf │ ├── scripts │ │ └── local │ │ │ └── kubeadm-token.sh │ ├── templates │ │ └── ccm-linode.yaml.template │ └── variables.tf └── nodes │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── outputs.tf ├── screens ├── dash-nodes.png └── dash-overview.png ├── scripts └── local │ ├── kubectl-conf.sh │ └── preflight.sh ├── terraform.sh ├── terraform.tf ├── variables.tf └── versions.tf /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | executors: 4 | terraform: 5 | docker: 6 | - image: hashicorp/terraform:0.12.24 7 | 8 | commands: 9 | install_build_deps: 10 | description: install all builds deps 11 | steps: 12 | - run: apk add --update --no-cache make ca-certificates openssh-keygen openssh-client bash python 13 | 14 | jobs: 15 | check: 16 | executor: terraform 17 | steps: 18 | - install_build_deps 19 | - checkout 20 | - run: 21 | name: lint 22 | command: make lint 23 | integration-test: 24 | parameters: 25 | ubuntu_version: 26 | type: string 27 | executor: terraform 28 | steps: 29 | - install_build_deps 30 | - checkout 31 | - run: echo -e 'y\n' | ssh-keygen -b 4096 -f $HOME/.ssh/id_rsa -t rsa -N '' 32 | - run: ssh-add $HOME/.ssh/id_rsa 33 | - run: wget https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl 34 | - run: 35 | name: integration test 36 | command: TF_VAR_cluster_name=test<< parameters.ubuntu_version >> TF_VAR_ubuntu_version=<< parameters.ubuntu_version >> make test 37 | - run: 38 | name: failure cleanup 39 | command: TF_VAR_cluster_name=test<< parameters.ubuntu_version >> TF_VAR_ubuntu_version=<< parameters.ubuntu_version >> make destroy 40 | when: on_fail 41 | 42 | workflows: 43 | version: 2 44 | main: 45 | jobs: 46 | - check 47 | - integration-test: 48 | matrix: 49 | parameters: 50 | ubuntu_version: ['16.04', '18.04', '20.04'] 51 | filters: 52 | branches: 53 | only: master 54 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | :+1::tada: First off, we appreciate you taking the time to contribute! THANK YOU! :tada::+1: 4 | 5 | We put together the handy guide below to help you get support for your work. Read on! 6 | 7 | ## I Just Want to Ask the Maintainers a Question 8 | 9 | The [Linode Community](https://www.linode.com/community/questions/) is a great place to get additional support. 10 | 11 | ## How Do I Submit A (Good) Bug Report or Feature Request 12 | 13 | Please open a [github issue](https://guides.github.com/features/issues/) to report bugs or suggest features. 14 | 15 | When filing an issue or feature request, help us avoid duplication and redundant effort -- check existing open or recently closed issues first. 16 | 17 | Detailed bug reports and requests are easier for us to work with. Please include the following in your issue: 18 | 19 | * A reproducible test case or series of steps 20 | * The version of our code being used 21 | * Any modifications you've made, relevant to the bug 22 | * Anything unusual about your environment or deployment 23 | * Screenshots and code samples where illustrative and helpful 24 | 25 | ## How to Open a Pull Request 26 | 27 | We follow the [fork and pull model](https://opensource.guide/how-to-contribute/#opening-a-pull-request) for open source contributions. 28 | 29 | Tips for a faster merge: 30 | * address one feature or bug per pull request. 31 | * large formatting changes make it hard for us to focus on your work. 32 | * follow language coding conventions. 33 | * make sure that tests pass. 34 | * make sure your commits are atomic, [addressing one change per commit](https://chris.beams.io/posts/git-commit/). 35 | * add tests! 36 | * save your large formatting changes for a different PR, so we can focus on your work 37 | * explain your rationale for why this feature is needed 38 | * link your PR to an [open issue](https://blog.github.com/2013-05-14-closing-issues-via-pull-requests/) 39 | 40 | ## Code of Conduct 41 | 42 | This project follows the [Linode Community Code of Conduct](https://www.linode.com/community/questions/conduct). 43 | 44 | ## Vulnerability Reporting 45 | 46 | If you discover a potential security issue in this project we ask that you notify Linode Security via our [vulnerability reporting process](https://hackerone.com/linode). Please do **not** create a public github issue. 47 | 48 | ## Licensing 49 | 50 | See the [LICENSE file](/LICENSE) for our project's licensing. 51 | 52 | 53 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## General: 2 | 3 | * [ ] Have you removed all sensitive information, including but not limited to access keys and passwords? 4 | * [ ] Have you checked to ensure there aren't other open or closed pull requests for the same bug/feature/question? 5 | 6 | ---- 7 | 8 | ## Feature Requests: 9 | * [ ] Have you explained your rationale for why this feature is needed? 10 | * [ ] Have you offered a proposed implementation/solution? 11 | 12 | ---- 13 | 14 | ## Bug Reporting 15 | 16 | ### Expected Behavior 17 | 18 | ### Actual Behavior 19 | 20 | ### Steps to Reproduce the Problem 21 | 22 | 1. 23 | 1. 24 | 1. 25 | 26 | ### Environment Specifications 27 | 28 | #### Screenshots, Code Blocks, and Logs 29 | 30 | #### Additional Notes 31 | 32 | ---- 33 | 34 | The [Linode Community](https://www.linode.com/community/questions/) is a great place to get additional support. 35 | 36 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### General: 2 | 3 | * [ ] Have you removed all sensitive information, including but not limited to access keys and passwords? 4 | * [ ] Have you checked to ensure there aren't other open or closed pull requests for the same bug/feature/question? 5 | * [ ] Have you followed our [pull request guidelines](CONTRIBUTING.md)? 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /.github/labels.yml: -------------------------------------------------------------------------------- 1 | - name: added-feature 2 | description: for new features in the changelog. 3 | color: a2eeef 4 | - name: changed 5 | description: for changes in existing functionality in the changelog. 6 | color: a2eeef 7 | - name: deprecated 8 | description: for soon-to-be removed features in the changelog. 9 | color: e4e669 10 | - name: removed 11 | description: for now removed features in the changelog. 12 | color: e4e669 13 | - name: bugfix 14 | description: for any bug fixes in the changelog. 15 | color: d73a4a 16 | - name: security 17 | description: for vulnerabilities in the changelog. 18 | color: dd4739 19 | - name: bug 20 | description: Something isn't working in this issue. 21 | color: d73a4a 22 | - name: enhancement 23 | description: New feature request in this issue. 24 | color: a2eeef 25 | - name: good first issue 26 | description: Low hanging fruit. 27 | color: 7057ff 28 | 29 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: 'v$NEXT_PATCH_VERSION' 2 | tag-template: 'v$NEXT_PATCH_VERSION' 3 | categories: 4 | - title: '🚀 Added' 5 | label: 'added-feature' 6 | - title: '🧰 Changed' 7 | label: 'changed' 8 | - title: "⚠️ Deprecated" 9 | label: "deprecated" 10 | - title: "⚠️ Removed" 11 | label: "removed" 12 | - title: '🐛 Bug Fixes' 13 | label: 'bugfix' 14 | - title: "⚠️ Security" 15 | label: "security" 16 | change-template: '- $TITLE @$AUTHOR (#$NUMBER)' 17 | no-changes-template: "- No changes" 18 | template: | 19 | ## Changes 20 | 21 | $CHANGES 22 | -------------------------------------------------------------------------------- /.github/workflows/label-sync.yml: -------------------------------------------------------------------------------- 1 | name: Sync labels 2 | on: 3 | push: 4 | branches: 5 | - master 6 | paths: 7 | - .github/labels.yml 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: micnncim/action-label-syncer@v1 14 | env: 15 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 16 | with: 17 | manifest: .github/labels.yml 18 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | update_release_draft: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: release-drafter/release-drafter@v5 13 | with: 14 | config-name: release-drafter.yml 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled files 2 | *.tfstate 3 | *.tfstate.backup 4 | *.terraform.tfstate.lock.info 5 | *.tfvars 6 | 7 | # Module directory 8 | .terraform/ 9 | 10 | # IDE 11 | .idea 12 | .vscode 13 | .DS_Store 14 | 15 | # K8S 16 | *.conf 17 | 18 | # Logs 19 | *.log 20 | 21 | -------------------------------------------------------------------------------- /.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/external" { 5 | version = "2.0.0" 6 | constraints = "~> 2.0.0" 7 | hashes = [ 8 | "h1:6S7hqjmUnoAZ5D/0F1VlJZKSJsUIBh7Ro0tLjGpKO0g=", 9 | "zh:07949780dd6a1d43e7b46950f6e6976581d9724102cb5388d3411a1b6f476bde", 10 | "zh:0a4f4636ff93f0644affa8474465dd8c9252946437ad025b28fc9f6603534a24", 11 | "zh:0dd7e05a974c649950d1a21d7015d3753324ae52ebdd1744b144bc409ca4b3e8", 12 | "zh:2b881032b9aa9d227ac712f614056d050bcdcc67df0dc79e2b2cb76a197059ad", 13 | "zh:38feb4787b4570335459ca75a55389df1a7570bdca8cdf5df4c2876afe3c14b4", 14 | "zh:40f7e0aaef3b1f4c2ca2bb1189e3fe9af8c296da129423986d1d99ccc8cfb86c", 15 | "zh:56b361f64f0f0df5c4f958ae2f0e6f8ba192f35b720b9d3ae1be068fabcf73d9", 16 | "zh:5fadb5880cd31c2105f635ded92b9b16f918c1dd989627a4ce62c04939223909", 17 | "zh:61fa0be9c14c8c4109cfb7be8d54a80c56d35dbae49d3231cddb59831e7e5a4d", 18 | "zh:853774bf97fbc4a784d5af5a4ca0090848430781ae6cfc586adeb48f7c44af79", 19 | ] 20 | } 21 | 22 | provider "registry.terraform.io/hashicorp/null" { 23 | version = "3.0.0" 24 | constraints = "~> 3.0.0" 25 | hashes = [ 26 | "h1:V1tzrSG6t3e7zWvUwRbGbhsWU2Jd/anrJpOl9XM+R/8=", 27 | "zh:05fb7eab469324c97e9b73a61d2ece6f91de4e9b493e573bfeda0f2077bc3a4c", 28 | "zh:1688aa91885a395c4ae67636d411475d0b831e422e005dcf02eedacaafac3bb4", 29 | "zh:24a0b1292e3a474f57c483a7a4512d797e041bc9c2fbaac42fe12e86a7fb5a3c", 30 | "zh:2fc951bd0d1b9b23427acc93be09b6909d72871e464088171da60fbee4fdde03", 31 | "zh:6db825759425599a326385a68acc6be2d9ba0d7d6ef587191d0cdc6daef9ac63", 32 | "zh:85985763d02618993c32c294072cc6ec51f1692b803cb506fcfedca9d40eaec9", 33 | "zh:a53186599c57058be1509f904da512342cfdc5d808efdaf02dec15f0f3cb039a", 34 | "zh:c2e07b49b6efa676bdc7b00c06333ea1792a983a5720f9e2233db27323d2707c", 35 | "zh:cdc8fe1096103cf5374751e2e8408ec4abd2eb67d5a1c5151fe2c7ecfd525bef", 36 | "zh:dbdef21df0c012b0d08776f3d4f34eb0f2f229adfde07ff252a119e52c0f65b7", 37 | ] 38 | } 39 | 40 | provider "registry.terraform.io/linode/linode" { 41 | version = "1.16.0" 42 | constraints = "~> 1.16.0" 43 | hashes = [ 44 | "h1:Fdep/ol4g9x+9TrYYd8RNTTEQop8K3gwWM2DhBWyR7U=", 45 | "zh:03c867440797b82012cd5d97f58fef5885dc0248683227299a39af836df222db", 46 | "zh:0486be7f72d6ea73d10140e23be8c1d2772b2d8be28c7bb39c73be83601405cf", 47 | "zh:181929d6880cac6500f4af1f3799385c47ccd69872cacf1042a3a48e445b2b02", 48 | "zh:18b7f6cc1ddf86e28322638607e1f84c1e9d56824c26903e22d4d12352f20b6e", 49 | "zh:4e65e7f9e17c334ff7047fc2dd8fc479c2509cba66834d89e2033a45e9275fe3", 50 | "zh:6077eda3fdf77a5158d9dc1a0c38492e23f7d679b1ac96382ba92ebe92e19266", 51 | "zh:642e7c96867c519176d84228a7f9104352212ae3c999b409eee1076b7ed90a96", 52 | "zh:6451f5117125fad9884214fe2f2635a2bed95912e64cf1c66a57c38558dfe907", 53 | "zh:83b957b30da19586393b9aea2cc93524a7d4c43dd07d11129a11d29c2b4bfb21", 54 | "zh:852954fe6cfe5278bd7c3d1079a9832bbf8c58436486489ed85154c0a0600633", 55 | "zh:a2385c51147a3c40707f7bfceb673c077e1054e8af6fb4c808cef56f995b8193", 56 | "zh:d21cd5cb5a635d18547430fe6cdfe3c6898541f9f3adc110edbf8d6e0439390d", 57 | ] 58 | } 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This project incorporates the following work, which has been permissibly forked subject to the following license: 2 | 3 | Modified work Copyright 2018 Linode 4 | Original work Copyright (c) 2017 Stefan Prodan 5 | 6 | 7 | Apache License 8 | Version 2.0, January 2004 9 | http://www.apache.org/licenses/ 10 | 11 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 12 | 13 | 1. Definitions. 14 | 15 | "License" shall mean the terms and conditions for use, reproduction, 16 | and distribution as defined by Sections 1 through 9 of this document. 17 | 18 | "Licensor" shall mean the copyright owner or entity authorized by 19 | the copyright owner that is granting the License. 20 | 21 | "Legal Entity" shall mean the union of the acting entity and all 22 | other entities that control, are controlled by, or are under common 23 | control with that entity. For the purposes of this definition, 24 | "control" means (i) the power, direct or indirect, to cause the 25 | direction or management of such entity, whether by contract or 26 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 27 | outstanding shares, or (iii) beneficial ownership of such entity. 28 | 29 | "You" (or "Your") shall mean an individual or Legal Entity 30 | exercising permissions granted by this License. 31 | 32 | "Source" form shall mean the preferred form for making modifications, 33 | including but not limited to software source code, documentation 34 | source, and configuration files. 35 | 36 | "Object" form shall mean any form resulting from mechanical 37 | transformation or translation of a Source form, including but 38 | not limited to compiled object code, generated documentation, 39 | and conversions to other media types. 40 | 41 | "Work" shall mean the work of authorship, whether in Source or 42 | Object form, made available under the License, as indicated by a 43 | copyright notice that is included in or attached to the work 44 | (an example is provided in the Appendix below). 45 | 46 | "Derivative Works" shall mean any work, whether in Source or Object 47 | form, that is based on (or derived from) the Work and for which the 48 | editorial revisions, annotations, elaborations, or other modifications 49 | represent, as a whole, an original work of authorship. For the purposes 50 | of this License, Derivative Works shall not include works that remain 51 | separable from, or merely link (or bind by name) to the interfaces of, 52 | the Work and Derivative Works thereof. 53 | 54 | "Contribution" shall mean any work of authorship, including 55 | the original version of the Work and any modifications or additions 56 | to that Work or Derivative Works thereof, that is intentionally 57 | submitted to Licensor for inclusion in the Work by the copyright owner 58 | or by an individual or Legal Entity authorized to submit on behalf of 59 | the copyright owner. For the purposes of this definition, "submitted" 60 | means any form of electronic, verbal, or written communication sent 61 | to the Licensor or its representatives, including but not limited to 62 | communication on electronic mailing lists, source code control systems, 63 | and issue tracking systems that are managed by, or on behalf of, the 64 | Licensor for the purpose of discussing and improving the Work, but 65 | excluding communication that is conspicuously marked or otherwise 66 | designated in writing by the copyright owner as "Not a Contribution." 67 | 68 | "Contributor" shall mean Licensor and any individual or Legal Entity 69 | on behalf of whom a Contribution has been received by Licensor and 70 | subsequently incorporated within the Work. 71 | 72 | 2. Grant of Copyright License. Subject to the terms and conditions of 73 | this License, each Contributor hereby grants to You a perpetual, 74 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 75 | copyright license to reproduce, prepare Derivative Works of, 76 | publicly display, publicly perform, sublicense, and distribute the 77 | Work and such Derivative Works in Source or Object form. 78 | 79 | 3. Grant of Patent License. Subject to the terms and conditions of 80 | this License, each Contributor hereby grants to You a perpetual, 81 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 82 | (except as stated in this section) patent license to make, have made, 83 | use, offer to sell, sell, import, and otherwise transfer the Work, 84 | where such license applies only to those patent claims licensable 85 | by such Contributor that are necessarily infringed by their 86 | Contribution(s) alone or by combination of their Contribution(s) 87 | with the Work to which such Contribution(s) was submitted. If You 88 | institute patent litigation against any entity (including a 89 | cross-claim or counterclaim in a lawsuit) alleging that the Work 90 | or a Contribution incorporated within the Work constitutes direct 91 | or contributory patent infringement, then any patent licenses 92 | granted to You under this License for that Work shall terminate 93 | as of the date such litigation is filed. 94 | 95 | 4. Redistribution. You may reproduce and distribute copies of the 96 | Work or Derivative Works thereof in any medium, with or without 97 | modifications, and in Source or Object form, provided that You 98 | meet the following conditions: 99 | 100 | (a) You must give any other recipients of the Work or 101 | Derivative Works a copy of this License; and 102 | 103 | (b) You must cause any modified files to carry prominent notices 104 | stating that You changed the files; and 105 | 106 | (c) You must retain, in the Source form of any Derivative Works 107 | that You distribute, all copyright, patent, trademark, and 108 | attribution notices from the Source form of the Work, 109 | excluding those notices that do not pertain to any part of 110 | the Derivative Works; and 111 | 112 | (d) If the Work includes a "NOTICE" text file as part of its 113 | distribution, then any Derivative Works that You distribute must 114 | include a readable copy of the attribution notices contained 115 | within such NOTICE file, excluding those notices that do not 116 | pertain to any part of the Derivative Works, in at least one 117 | of the following places: within a NOTICE text file distributed 118 | as part of the Derivative Works; within the Source form or 119 | documentation, if provided along with the Derivative Works; or, 120 | within a display generated by the Derivative Works, if and 121 | wherever such third-party notices normally appear. The contents 122 | of the NOTICE file are for informational purposes only and 123 | do not modify the License. You may add Your own attribution 124 | notices within Derivative Works that You distribute, alongside 125 | or as an addendum to the NOTICE text from the Work, provided 126 | that such additional attribution notices cannot be construed 127 | as modifying the License. 128 | 129 | You may add Your own copyright statement to Your modifications and 130 | may provide additional or different license terms and conditions 131 | for use, reproduction, or distribution of Your modifications, or 132 | for any such Derivative Works as a whole, provided Your use, 133 | reproduction, and distribution of the Work otherwise complies with 134 | the conditions stated in this License. 135 | 136 | 5. Submission of Contributions. Unless You explicitly state otherwise, 137 | any Contribution intentionally submitted for inclusion in the Work 138 | by You to the Licensor shall be under the terms and conditions of 139 | this License, without any additional terms or conditions. 140 | Notwithstanding the above, nothing herein shall supersede or modify 141 | the terms of any separate license agreement you may have executed 142 | with Licensor regarding such Contributions. 143 | 144 | 6. Trademarks. This License does not grant permission to use the trade 145 | names, trademarks, service marks, or product names of the Licensor, 146 | except as required for reasonable and customary use in describing the 147 | origin of the Work and reproducing the content of the NOTICE file. 148 | 149 | 7. Disclaimer of Warranty. Unless required by applicable law or 150 | agreed to in writing, Licensor provides the Work (and each 151 | Contributor provides its Contributions) on an "AS IS" BASIS, 152 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 153 | implied, including, without limitation, any warranties or conditions 154 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 155 | PARTICULAR PURPOSE. You are solely responsible for determining the 156 | appropriateness of using or redistributing the Work and assume any 157 | risks associated with Your exercise of permissions under this License. 158 | 159 | 8. Limitation of Liability. In no event and under no legal theory, 160 | whether in tort (including negligence), contract, or otherwise, 161 | unless required by applicable law (such as deliberate and grossly 162 | negligent acts) or agreed to in writing, shall any Contributor be 163 | liable to You for damages, including any direct, indirect, special, 164 | incidental, or consequential damages of any character arising as a 165 | result of this License or out of the use or inability to use the 166 | Work (including but not limited to damages for loss of goodwill, 167 | work stoppage, computer failure or malfunction, or any and all 168 | other commercial damages or losses), even if such Contributor 169 | has been advised of the possibility of such damages. 170 | 171 | 9. Accepting Warranty or Additional Liability. While redistributing 172 | the Work or Derivative Works thereof, You may choose to offer, 173 | and charge a fee for, acceptance of support, warranty, indemnity, 174 | or other liability obligations and/or rights consistent with this 175 | License. However, in accepting such obligations, You may act only 176 | on Your own behalf and on Your sole responsibility, not on behalf 177 | of any other Contributor, and only if You agree to indemnify, 178 | defend, and hold each Contributor harmless for any liability 179 | incurred by, or claims asserted against, such Contributor by reason 180 | of your accepting any such warranty or additional liability. 181 | 182 | END OF TERMS AND CONDITIONS 183 | 184 | APPENDIX: How to apply the Apache License to your work. 185 | 186 | To apply the Apache License to your work, attach the following 187 | boilerplate notice, with the fields enclosed by brackets "[]" 188 | replaced with your own identifying information. (Don't include 189 | the brackets!) The text should be enclosed in the appropriate 190 | comment syntax for the file format. We also recommend that a 191 | file or class name and description of purpose be included on the 192 | same "printed page" as the copyright notice for easier 193 | identification within third-party archives. 194 | 195 | Copyright 2018 Linode 196 | 197 | Licensed under the Apache License, Version 2.0 (the "License"); 198 | you may not use this file except in compliance with the License. 199 | You may obtain a copy of the License at 200 | 201 | http://www.apache.org/licenses/LICENSE-2.0 202 | 203 | Unless required by applicable law or agreed to in writing, software 204 | distributed under the License is distributed on an "AS IS" BASIS, 205 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 206 | See the License for the specific language governing permissions and 207 | limitations under the License. 208 | 209 | APPENDIX II: 210 | 211 | Copyright 2017 Stefan Prodan 212 | 213 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 214 | 215 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 216 | 217 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 218 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: init plan apply destroy test 2 | 3 | .EXPORT_ALL_VARIABLES: 4 | 5 | TF_INPUT = 0 6 | TF_WORKSPACE = testing 7 | TF_IN_AUTOMATION = 1 8 | TF_VAR_nodes = 1 9 | TF_VAR_linode_token = ${LINODE_TOKEN} 10 | 11 | init: 12 | terraform init 13 | 14 | lint: 15 | terraform fmt -recursive -check -diff . 16 | 17 | plan: check-token 18 | terraform plan 19 | 20 | apply: check-token 21 | terraform apply -auto-approve 22 | 23 | destroy: check-token 24 | terraform destroy -auto-approve 25 | 26 | test: lint init plan apply destroy 27 | 28 | check-token: 29 | @if test "$(LINODE_TOKEN)" = "" ; then \ 30 | echo "LINODE_TOKEN must be set"; \ 31 | exit 1; \ 32 | fi 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Terraform installer for Linode Instances 2 | 3 | :warning: | This project is experimental and should not be used in production environments. 4 | :---: | :--- 5 | 6 | 7 | This Terraform module creates a Kubernetes Cluster on Linode Cloud infrastructure running Ubuntu. The cluster is designed to take advantage of the Linode regional private network, and is equiped with Linode specific cluster enhancements. 8 | 9 | Cluster size and instance types are configurable through Terraform variables. 10 | 11 | ## Validated configurations 12 | 13 | k8s_version|docker_version|cni_version|crictl_version|k8s_feature_gates 14 | ---|---|---|---|--- 15 | v1.19.6|19.03|v0.8.7|v1.15.0| 16 | v1.18.13|19.03|v0.8.7|v1.15.0| 17 | v1.17.16|19.03|v0.8.7|v1.15.0| 18 | v1.16.15|19.03|v0.8.7|v1.15.0| 19 | 20 | ## Install 21 | 22 | ### Prerequisites 23 | 24 | * Terraform must be installed 25 | * Bash must be installed 26 | * SSH should be installed and configured with an SSH Key and Agent (Recommended) 27 | * Having kubectl installed is recommended 28 | 29 | Note that you'll need Terraform v0.10 or newer to run this project. 30 | 31 | ### Linode API Token 32 | 33 | Before running the project you'll have to create an access token for Terraform to connect to the Linode API. 34 | Using the token and your access key, create the `LINODE_TOKEN` environment variable: 35 | 36 | ```bash 37 | read -sp "Linode Token: " TF_VAR_linode_token # Enter your Linode Token (it will be hidden) 38 | export TF_VAR_linode_token 39 | ``` 40 | 41 | This variable will need to be supplied to every Terraform `apply`, `plan`, and `destroy` command using `-var linode_token=$LINODE_TOKEN` unless a `terraform.tfvars` file is created with this secret token. 42 | 43 | ## Usage 44 | 45 | Create a `main.tf` file in a new directory with the following contents: 46 | 47 | ```hcl 48 | module "k8s" { 49 | source = "linode/k8s/linode" 50 | 51 | linode_token = "YOUR TOKEN HERE" 52 | } 53 | ``` 54 | 55 | That's all it takes to get started! 56 | 57 | Pin to a specific module version using `version = "..."` to avoid upgrading to a version with breaking changes. Upgrades to this module could potentially replace all master and worker nodes resulting in data loss. The `terraform plan` will report this, but it may not be obvious. 58 | 59 | ```hcl 60 | module "k8s" { 61 | source = "linode/k8s/linode" 62 | version = "0.1.0" 63 | 64 | linode_token = "YOUR TOKEN HERE" 65 | } 66 | ``` 67 | 68 | Choose a Terraform workspace name (because the default is `default`). In this example we've chosen `linode`. The workspace name will be used as a prefix for Linode resource created in this cluster, for example: `linode-master-1`, `linode-node-1`. Alternate workspaces can be created and selected to change clusters. 69 | 70 | ```bash 71 | terraform workspace new linode 72 | ``` 73 | 74 | Create an Linode Kubernetes cluster with one master and a node: 75 | 76 | ```bash 77 | terraform apply \ 78 | -var region=eu-west \ 79 | -var server_type_master=g6-standard-2 \ 80 | -var nodes=1 \ 81 | -var server_type_node=g6-standard-2 \ 82 | ``` 83 | 84 | This will do the following: 85 | 86 | * provisions Linode Instances in parallel with Ubuntu 20.04 (the Linode instance type/size of the `master` and the `node` may be different) 87 | * connects to the Linode Instances via SSH and installs various Kubernetes components and supporting binaries: [kubeadm](https://kubernetes.io/docs/reference/setup-tools/kubeadm/), [kubectl](https://kubernetes.io/docs/reference/kubectl/overview/), [kubelet](https://kubernetes.io/docs/concepts/overview/components/#kubelet), [kubernetes-cni](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#cni), and [crictl](https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/). 88 | * installs a Calico network between Linode Instances 89 | * runs kubeadm init on the master server and configures kubectl 90 | * joins the nodes in the cluster using the kubeadm token obtained from the master 91 | * installs Linode add-ons: 92 | * [CSI](https://github.com/linode/linode-blockstorage-csi-driver) (LinodeBlock Storage Volumes) 93 | * [CCM](https://github.com/linode/linode-cloud-controller-manager) (Linode NodeBalancers) 94 | * [External-DNS](https://github.com/kubernetes-incubator/external-dns/blob/master/docs/tutorials/linode.md) (Linode Domains) 95 | * installs cluster add-ons: 96 | * [Kubernetes dashboard](https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/) 97 | * [Metrics Server](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-metrics-pipeline/#metrics-server) 98 | * copies the kubectl admin config file for local `kubectl` use via the public IP of the API server 99 | 100 | A full list of the supported variables are available in the [Terraform Module Registry](https://registry.terraform.io/modules/linode/k8s/linode/?tab=inputs). 101 | 102 | After applying the Terraform plan you'll see several output variables like the master public IP, 103 | the `kubeadmn join` command and the current workspace admin config (for use with `kubectl`). 104 | 105 | The cluster node count can be scaled up by increasing the number of Linode Instances acting as nodes: 106 | 107 | ```bash 108 | terraform apply -var nodes=3 109 | ``` 110 | 111 | Tear down the whole infrastructure with: 112 | 113 | ```bash 114 | terraform destroy -force 115 | ``` 116 | 117 | Be sure to clean-up any CSI created Block Storage Volumes, and CCM created NodeBalancers that you no longer require. 118 | 119 | ### Remote control 120 | 121 | The `kubectl` config file format is `.conf` as in `linode.conf`. Kubectl will use this file when provided through `--kubeconfig` or when set in the `KUBECONFIG` environment variable. 122 | 123 | If you have `kubectl` install locally, you can use it to work with your Linode cluster. You can always ssh into the master Linode Instance and run `kubectl` there (without the `--kubeconfig` option or environment variable). 124 | 125 | ```bash 126 | $ export KUBECONFIG="$(pwd)/$(terraform output kubectl_config)" 127 | $ kubectl top nodes 128 | 129 | NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% 130 | linode-master-1 655m 16% 873Mi 45% 131 | linode-node-1 147m 3% 618Mi 32% 132 | linode-node-2 101m 2% 584Mi 30% 133 | ``` 134 | 135 | In order to access the dashboard locally, you can use `kubectl proxy` then browse to 136 | 137 | ```bash 138 | $ kubectl proxy & 139 | [1] 37630 140 | Starting to serve on 127.0.0.1:8001 141 | ``` 142 | 143 | To authenticate, provide the [kubeconfig file or generate a token](https://github.com/kubernetes/dashboard/wiki/Access-control#authentication). For demonstrative purposes, an existing system token can be used. This is not recommended for production clusters. 144 | 145 | ```bash 146 | kubectl -n kube-system describe secrets `kubectl -n kube-system get secrets | awk '/clusterrole-aggregation-controller/ {print $1}'` | awk '/token:/ {print $2}' 147 | ``` 148 | 149 | ![Overview](https://github.com/linode/terraform-linode-k8s/blob/master/screens/dash-overview.png) 150 | 151 | ![Nodes](https://github.com/linode/terraform-linode-k8s/blob/master/screens/dash-nodes.png) 152 | 153 | ## Addons Included 154 | 155 | ### [**Linode Cloud Controller Manager (CCM)**](https://github.com/linode/linode-cloud-controller-manager) 156 | 157 | A primary function of the CCM is to register and maintain Kubernetes `LoadBalancer` settings within a Linode [`NodeBalancer`](https://www.linode.com/nodebalancers). This is needed to allow traffic from the Internet into the cluster in the most fault tollerant way (obviously very important!) 158 | 159 | The CCM also annotates new Kubernetes Nodes with Linode specific details, including the LinodeID and instance type. Linode hostnames and network addresses are automatically associated with their corresponding Kubernetes resources, forming the basis for a variety of Kubernetes features. 160 | 161 | The CCM monitors the Linode API for changes in the Linode instance and will remove a Kubernetes Node if it finds the Linode has been deleted. Resources will automatically be re-scheduled if the Linode is powered off. 162 | 163 | [Learn more about the CCM concept on kubernetes.io.](https://kubernetes.io/docs/concepts/architecture/cloud-controller/) 164 | 165 | ### [**Linode Container Storage Interface (CSI)**](https://github.com/linode/linode-blockstorage-csi-driver) 166 | 167 | The CSI provides a Kubernetes `Storage Class` which can be used to create `Persistent Volumes` (PV) using [Linode Block Storage Volumes](https://www.linode.com/blockstorage). Pods then create `Persistent Volume Claims` (PVC) to attach to these volumes. 168 | 169 | When a `PV` is deleted, the Linode Block Storage Volume will be deleted as well, based on the `ReclaimPolicy`. 170 | 171 | In this Terraform Module, the `DefaultStorageClass` is provided by the `Linode CSI`. Persistent volumes can be defined with an alternate `storageClass`. 172 | 173 | [Learn More about Persistent Volumes on kubernetes.io.](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) 174 | 175 | ### [**External-DNS support for Linode**](https://github.com/kubernetes-incubator/external-dns/blob/master/docs/tutorials/linode.md) 176 | 177 | Unlike [CoreDNS](https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/) (or KubeDNS), which provides DNS services within the Kubernetes cluster, [External-DNS](https://github.com/kubernetes-incubator/external-dns/blob/master/README.md) publishes the public facing IP addresses associated with exposed services to a public DNS server, such as the [Linode DNS Manager](https://www.linode.com/dns-manager). 178 | 179 | As configured in this Terraform module, any service or ingress with a specific annotation, will have a DNS record managed for it, pointing to the appropriate Linode or NodeBalancer IP address. The domain must already be configured in the [Linode DNS Manager](https://www.linode.com/docs/platform/manager/dns-manager/#domain-zones). 180 | 181 | [Learn more at the External-DNS Github project.](https://github.com/kubernetes-incubator/external-dns) 182 | 183 | ## Development 184 | 185 | To make changes to this project, verify that you have the prerequisites and then clone the repo. Instead of using the Terraform `module` syntax, and being confined by the variables that are provided, you'll be able to make any changes necessary. 186 | 187 | ```bash 188 | git clone https://github.com/linode/terraform-linode-k8s.git 189 | cd terraform-linode-k8s 190 | ``` 191 | 192 | Or if you won't be submitting changes, you can use `terraform init`: 193 | 194 | ```bash 195 | terraform init --from-module=linode/k8s/linode linode-k8s 196 | ``` 197 | 198 | ### Modules 199 | 200 | This terraform modules is composed of three sub-modules for reuse and separation of concerns. 201 | 202 | * Instance - Provisions a base Linode Instance for the cluster. 203 | * Master - Uses the Instance module as a base and futher provisions a Kubernetes control-plane. 204 | * Node - Uses the Instance module as a base and further provisions a Kubernetes worker joined to a control-plane using module parameters. 205 | 206 | ### Contribution Guidelines 207 | 208 | Would you like to improve the `terraform-linode-k8s` module? Please start [here](https://github.com/linode/terraform-linode-k8s/blob/master/.github/CONTRIBUTING.md). 209 | 210 | ### Join us on Slack 211 | 212 | For general help or discussion, join the [Kubernetes Slack](http://slack.k8s.io/) channel [#linode](https://kubernetes.slack.com/messages/CD4B15LUR). 213 | -------------------------------------------------------------------------------- /config/sshd_config: -------------------------------------------------------------------------------- 1 | # Package generated configuration file 2 | # See the sshd_config(5) manpage for details 3 | 4 | # What ports, IPs and protocols we listen for 5 | Port 22 6 | # Use these options to restrict which interfaces/protocols sshd will bind to 7 | #ListenAddress :: 8 | #ListenAddress 0.0.0.0 9 | Protocol 2 10 | # HostKeys for protocol version 2 11 | HostKey /etc/ssh/ssh_host_rsa_key 12 | HostKey /etc/ssh/ssh_host_dsa_key 13 | HostKey /etc/ssh/ssh_host_ecdsa_key 14 | HostKey /etc/ssh/ssh_host_ed25519_key 15 | #Privilege Separation is turned on for security 16 | UsePrivilegeSeparation yes 17 | 18 | # Lifetime and size of ephemeral version 1 server key 19 | KeyRegenerationInterval 3600 20 | ServerKeyBits 1024 21 | 22 | # Logging 23 | SyslogFacility AUTH 24 | LogLevel INFO 25 | 26 | # Authentication: 27 | LoginGraceTime 120 28 | PermitRootLogin yes 29 | StrictModes yes 30 | 31 | RSAAuthentication yes 32 | PubkeyAuthentication yes 33 | #AuthorizedKeysFile %h/.ssh/authorized_keys 34 | 35 | # Don't read the user's ~/.rhosts and ~/.shosts files 36 | IgnoreRhosts yes 37 | # For this to work you will also need host keys in /etc/ssh_known_hosts 38 | RhostsRSAAuthentication no 39 | # similar for protocol version 2 40 | HostbasedAuthentication no 41 | # Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication 42 | #IgnoreUserKnownHosts yes 43 | 44 | # To enable empty passwords, change to yes (NOT RECOMMENDED) 45 | PermitEmptyPasswords no 46 | 47 | # Change to yes to enable challenge-response passwords (beware issues with 48 | # some PAM modules and threads) 49 | ChallengeResponseAuthentication no 50 | 51 | # Change to no to disable tunnelled clear text passwords 52 | PasswordAuthentication yes 53 | 54 | # Kerberos options 55 | #KerberosAuthentication no 56 | #KerberosGetAFSToken no 57 | #KerberosOrLocalPasswd yes 58 | #KerberosTicketCleanup yes 59 | 60 | # GSSAPI options 61 | #GSSAPIAuthentication no 62 | #GSSAPICleanupCredentials yes 63 | 64 | X11Forwarding yes 65 | X11DisplayOffset 10 66 | PrintMotd no 67 | PrintLastLog yes 68 | TCPKeepAlive yes 69 | #UseLogin no 70 | 71 | #MaxStartups 10:30:60 72 | #Banner /etc/issue.net 73 | 74 | # Allow client to pass locale environment variables 75 | AcceptEnv LANG LC_* 76 | 77 | Subsystem sftp /usr/lib/openssh/sftp-server 78 | 79 | # Set this to 'yes' to enable PAM authentication, account processing, 80 | # and session processing. If this is enabled, PAM authentication will 81 | # be allowed through the ChallengeResponseAuthentication and 82 | # PasswordAuthentication. Depending on your PAM configuration, 83 | # PAM authentication via ChallengeResponseAuthentication may bypass 84 | # the setting of "PermitRootLogin without-password". 85 | # If you just want the PAM account and session checks to run without 86 | # PAM authentication, then enable this but set PasswordAuthentication 87 | # and ChallengeResponseAuthentication to 'no'. 88 | UsePAM yes 89 | 90 | # Customizations 91 | # See https://github.com/hashicorp/terraform/issues/18517#issuecomment-415023605 92 | ClientAliveInterval 120 93 | ClientAliveCountMax 720 -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | provider "linode" { 2 | token = var.linode_token 3 | } 4 | 5 | resource "null_resource" "preflight-checks" { 6 | # Force re-run 7 | triggers = { 8 | key = uuid() 9 | } 10 | 11 | provisioner "local-exec" { 12 | command = "${path.cwd}/${path.module}/scripts/local/preflight.sh \"${var.ccm_image}\" \"${var.csi_manifest}\" \"${var.calico_manifest}\"" 13 | working_dir = "${path.cwd}/${path.module}" 14 | } 15 | } 16 | 17 | module "masters" { 18 | source = "./modules/masters" 19 | label_prefix = var.cluster_name == "" ? terraform.workspace : var.cluster_name 20 | node_class = "master" 21 | node_count = var.masters 22 | node_type = var.server_type_master 23 | linode_token = var.linode_token 24 | 25 | ubuntu_version = var.ubuntu_version 26 | docker_version = var.docker_version 27 | k8s_version = var.k8s_version 28 | crictl_version = var.crictl_version 29 | k8s_feature_gates = var.k8s_feature_gates 30 | cni_version = var.cni_version 31 | ssh_public_key = var.ssh_public_key 32 | region = var.region 33 | linode_group = var.cluster_name 34 | 35 | //todo variable instead of workspace? 36 | cluster_name = var.cluster_name == "" ? terraform.workspace : var.cluster_name 37 | } 38 | 39 | module "nodes" { 40 | source = "./modules/nodes" 41 | label_prefix = var.cluster_name == "" ? terraform.workspace : var.cluster_name 42 | node_class = "node" 43 | node_count = var.nodes 44 | node_type = var.server_type_node 45 | 46 | ubuntu_version = var.ubuntu_version 47 | docker_version = var.docker_version 48 | k8s_version = var.k8s_version 49 | crictl_version = var.crictl_version 50 | k8s_feature_gates = var.k8s_feature_gates 51 | cni_version = var.cni_version 52 | ssh_public_key = var.ssh_public_key 53 | region = var.region 54 | linode_group = var.cluster_name 55 | kubeadm_join_command = module.masters.kubeadm_join_command 56 | } 57 | 58 | resource "null_resource" "local_kubectl" { 59 | // todo 60 | depends_on = [module.masters] 61 | 62 | provisioner "local-exec" { 63 | command = "${path.cwd}/${path.module}/scripts/local/kubectl-conf.sh ${terraform.workspace} ${module.masters.k8s_master_public_ip} ${module.masters.k8s_master_private_ip} ${var.ssh_public_key}" 64 | on_failure = continue 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /modules/instances/README.md: -------------------------------------------------------------------------------- 1 | ## Ubuntu Kubernetes Instances 2 | 3 | This module provisions a Linode Instance using Ubuntu while staging some Kubernetes tooling. 4 | -------------------------------------------------------------------------------- /modules/instances/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | root_user = "root" 3 | 4 | ubuntu_images = { 5 | "16.04" : "linode/ubuntu16.04lts", 6 | "18.04" : "linode/ubuntu18.04", 7 | "20.04" : "linode/ubuntu20.04", 8 | } 9 | } 10 | 11 | resource "linode_instance" "instance" { 12 | count = var.node_count 13 | region = var.region 14 | label = "${var.label_prefix == "" ? "" : "${var.label_prefix}-"}${var.node_class}-${count.index + 1}" 15 | group = var.linode_group 16 | type = var.node_type 17 | private_ip = var.private_ip 18 | 19 | authorized_keys = [chomp(file(var.ssh_public_key))] 20 | image = local.ubuntu_images[var.ubuntu_version] 21 | 22 | provisioner "remote-exec" { 23 | inline = [ 24 | "mkdir -p /root/init/", 25 | ] 26 | 27 | connection { 28 | host = self.ip_address 29 | user = local.root_user 30 | timeout = "300s" 31 | } 32 | } 33 | 34 | provisioner "file" { 35 | source = "${path.cwd}/${path.module}/scripts/" 36 | destination = "/root/init/" 37 | 38 | connection { 39 | host = self.ip_address 40 | user = local.root_user 41 | timeout = "300s" 42 | } 43 | } 44 | 45 | provisioner "remote-exec" { 46 | inline = [ 47 | "set -e", 48 | "chmod +x /root/init/start.sh && sudo /root/init/start.sh", 49 | "chmod +x /root/init/linode-network.sh && sudo /root/init/linode-network.sh ${self.private_ip_address} ${self.label}", 50 | "chmod +x /root/init/kubeadm-install.sh && sudo /root/init/kubeadm-install.sh \"${var.k8s_version}\" \"${var.cni_version}\" \"${var.crictl_version}\" \"${self.label}\" \"${var.use_public ? self.ip_address : self.private_ip_address}\" \"${var.k8s_feature_gates}\" \"${var.docker_version}\"", 51 | ] 52 | 53 | connection { 54 | host = self.ip_address 55 | user = local.root_user 56 | timeout = "300s" 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /modules/instances/outputs.tf: -------------------------------------------------------------------------------- 1 | // todo: ha, return nb address 2 | output "public_ip_address" { 3 | depends_on = [linode_instance.instance[0]] 4 | description = "Public IP Address of the first instance in the group" 5 | value = element(concat(linode_instance.instance.*.ip_address, [""]), 0) 6 | } 7 | 8 | // todo: this doesnt make sense in ha -- return all? 9 | output "private_ip_address" { 10 | description = "Private IP Address of the first instance in the group" 11 | depends_on = [linode_instance.instance[0]] 12 | value = element(concat(linode_instance.instance.*.private_ip_address, [""]), 0) 13 | } 14 | 15 | output "nodes_public_ip" { 16 | depends_on = [linode_instance.instance] 17 | description = "Public IP Address of the instance(s)" 18 | value = concat(linode_instance.instance.*.ip_address) 19 | } 20 | -------------------------------------------------------------------------------- /modules/instances/scripts/end.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | -------------------------------------------------------------------------------- /modules/instances/scripts/kubeadm-init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | K8S_CLUSTERNAME="$1" 5 | K8S_VERSION="$2" 6 | NODE_PRIVATE_IP="$3" 7 | NODE_PUBLIC_IP="$4" 8 | K8S_FEATURE_GATES="$5" 9 | POD_NETWORK="10.244.0.0/16" 10 | 11 | kubeadm init \ 12 | --apiserver-advertise-address "$NODE_PUBLIC_IP" \ 13 | --apiserver-cert-extra-sans "$NODE_PRIVATE_IP" \ 14 | --pod-network-cidr "$POD_NETWORK" \ 15 | --kubernetes-version "$K8S_VERSION" \ 16 | --cri-socket /var/run/dockershim.sock \ 17 | --feature-gates "$K8S_FEATURE_GATES" 18 | -------------------------------------------------------------------------------- /modules/instances/scripts/kubeadm-install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o nounset -o errexit -o pipefail 3 | 4 | K8S_VERSION=$1 5 | CNI_VERSION=$2 6 | CRICTL_VERSION="$3" 7 | HOSTNAME=$4 8 | NODE_IP=$5 9 | K8S_FEATURE_GATES="$6" 10 | DOCKER_VERSION=$7 11 | 12 | get_latest_pkg_revision() { 13 | local pkg="$1" 14 | local filter="$2" 15 | 16 | version=`apt list -a $pkg | grep "$filter" | head -1 | awk '{print $2}'` 17 | echo $version 18 | } 19 | 20 | # disable swap 21 | sudo swapoff -a 22 | sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab 23 | 24 | # add kubelet configuration 25 | cat << EOF > /etc/default/kubelet 26 | KUBELET_EXTRA_ARGS="--cloud-provider=external --feature-gates=${K8S_FEATURE_GATES}" 27 | EOF 28 | 29 | # make sure we have Internet connectivity before proceeding 30 | n=0 31 | until [ $n -ge 120 ] 32 | do 33 | curl -L "https://github.com" >/dev/null 2>&1 && curl -L "https://storage.googleapis.com" >/dev/null 2>&1 && break 34 | n=$[$n+1] 35 | sleep 1 36 | done 37 | 38 | mkdir -p /etc/kubernetes/manifests 39 | 40 | sudo modprobe br_netfilter 41 | 42 | # configure ip tables to see bridge traffic 43 | cat </dev/null "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz 86 | 87 | # add kubernetes apt source 88 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - 89 | cat <&1 >/dev/null; echo $?) 18 | ret=$(echo $out | tail -n1) 19 | 20 | attempts=$((attempts+1)) 21 | if [[ $ret -eq 0 ]]; then 22 | echo "successfully applied manifest: $manifest" 23 | return 0 24 | elif [[ ( "$out" =~ "no matches for kind" ) && ( "$attempts" -lt 2 ) ]]; then 25 | echo "applying $manifest failed because a CRD was not yet published." 26 | echo "retrying in 2s..." 27 | sleep 2 28 | continue 29 | else 30 | echo "failed to apply manifest: $manifest" 31 | echo "exit code: $ret" 32 | echo "output: $out" 33 | return 1 34 | fi 35 | done 36 | } 37 | 38 | sed -i -E \ 39 | -e 's/\$\(LINODE_REGION\)/'$LINODE_REGION'/g' \ 40 | -e 's/\$\(LINODE_TOKEN\)/'$LINODE_TOKEN'/g' \ 41 | /root/init/linode-token.yaml 42 | 43 | # TODO swap these for helm charts 44 | for yaml in \ 45 | linode-token.yaml \ 46 | ccm-linode.yaml \ 47 | csi-linode.yaml \ 48 | external-dns.yaml \ 49 | ; do apply_manifest /root/init/${yaml}; done 50 | 51 | rm /root/init/linode-token.yaml 52 | -------------------------------------------------------------------------------- /modules/instances/scripts/linode-network.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o nounset -o errexit 3 | 4 | NODE_PRIVATE_IP="$1" 5 | HOSTNAME="$2" 6 | 7 | sed -E "/localhost$/s/(localhost)$/\1 $HOSTNAME/" -i /etc/hosts 8 | 9 | echo -e "\nAddress=${NODE_PRIVATE_IP}/17" >> /etc/systemd/network/05-eth0.network 10 | systemctl daemon-reload 11 | systemctl restart systemd-networkd 12 | 13 | hostnamectl set-hostname ${HOSTNAME} && sudo hostname -F /etc/hostname 14 | -------------------------------------------------------------------------------- /modules/instances/scripts/monitoring-install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # TODO swap these for helm charts 6 | 7 | kubectl apply -f /root/init/dashboard-rbac.yaml 8 | kubectl apply -f /root/init/dashboard.yaml 9 | kubectl apply -f /root/init/metrics-server.yaml 10 | -------------------------------------------------------------------------------- /modules/instances/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | for mod in ip_vs_sh ip_vs ip_vs_rr ip_vs_wrr nf_conntrack_ipv4; do echo $mod | sudo tee /etc/modules-load.d/$mod.conf; done 5 | -------------------------------------------------------------------------------- /modules/instances/variables.tf: -------------------------------------------------------------------------------- 1 | variable "node_count" { 2 | default = "1" 3 | description = "Number of Kubernetes Nodes to provision" 4 | } 5 | 6 | variable "node_class" { 7 | default = "node" 8 | description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)" 9 | } 10 | 11 | variable "node_type" { 12 | default = "g6-standard-4" 13 | description = "Linode Instance type for nodes" 14 | } 15 | 16 | variable "private_ip" { 17 | default = true 18 | description = "Enables Linode Instance Private IP addresses" 19 | } 20 | 21 | variable "label_prefix" { 22 | default = "" 23 | description = "Linode label prefix" 24 | } 25 | 26 | variable "use_public" { 27 | description = "Use the public network interface" 28 | default = false 29 | } 30 | 31 | variable "cni_version" { 32 | description = "Container Network Plugin Version" 33 | } 34 | 35 | variable "ssh_public_key" {} 36 | 37 | variable "region" { 38 | description = "Linode Region: us-central us-west us-southeast us-east eu-west ap-south eu-central ap-northeast ap-northeast-1a" 39 | } 40 | 41 | variable "linode_group" { 42 | default = "" 43 | description = "Linode display group for provisioned instances" 44 | } 45 | 46 | variable "k8s_version" { 47 | description = "Kubernetes version to install" 48 | } 49 | 50 | variable "crictl_version" { 51 | description = "Container Runtime Interface version to install" 52 | } 53 | 54 | variable "k8s_feature_gates" { 55 | description = "Feature gates to enable in the Kubelet and API server" 56 | } 57 | 58 | variable "ubuntu_version" { 59 | description = "Ubuntu version to install" 60 | default = "16.04" 61 | 62 | validation { 63 | condition = contains(["16.04", "18.04", "20.04"], var.ubuntu_version) 64 | error_message = "Ubuntu version must be one of supported: 16.04, 18.04, or 20.04." 65 | } 66 | } 67 | 68 | variable "docker_version" { 69 | description = "The docker version to install" 70 | } 71 | -------------------------------------------------------------------------------- /modules/instances/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | linode = { 4 | source = "linode/linode" 5 | } 6 | } 7 | required_version = ">= 0.13" 8 | } 9 | -------------------------------------------------------------------------------- /modules/masters/README.md: -------------------------------------------------------------------------------- 1 | ## Ubuntu Kubernetes Master Instances 2 | 3 | This module provisions a Kubernetes Master Linode Instance using Ubuntu. 4 | -------------------------------------------------------------------------------- /modules/masters/main.tf: -------------------------------------------------------------------------------- 1 | module "master_instance" { 2 | source = "../instances" 3 | label_prefix = var.label_prefix 4 | node_type = var.node_type 5 | node_count = "1" // HA not supported yet 6 | node_class = "master" 7 | linode_group = var.linode_group 8 | private_ip = "true" 9 | use_public = "true" // rename this var, sent to kubeadm 10 | 11 | ubuntu_version = var.ubuntu_version 12 | k8s_version = var.k8s_version 13 | k8s_feature_gates = var.k8s_feature_gates 14 | cni_version = var.cni_version 15 | crictl_version = var.crictl_version 16 | ssh_public_key = var.ssh_public_key 17 | region = var.region 18 | docker_version = var.docker_version 19 | } 20 | 21 | resource "null_resource" "masters_provisioner" { 22 | depends_on = [module.master_instance] 23 | 24 | provisioner "remote-exec" { 25 | inline = [ 26 | "mkdir -p /root/init/", 27 | ] 28 | 29 | connection { 30 | user = "root" 31 | timeout = "300s" 32 | host = module.master_instance.public_ip_address 33 | } 34 | } 35 | 36 | provisioner "file" { 37 | source = "${path.cwd}/${path.module}/manifests/" 38 | destination = "/root/init/" 39 | 40 | connection { 41 | user = "root" 42 | timeout = "300s" 43 | host = module.master_instance.public_ip_address 44 | } 45 | } 46 | 47 | provisioner "remote-exec" { 48 | # TODO advertise on public adress 49 | inline = [ 50 | "set -e", 51 | "chmod +x /root/init/kubeadm-init.sh && sudo /root/init/kubeadm-init.sh \"${var.cluster_name}\" \"${var.k8s_version}\" \"${module.master_instance.public_ip_address}\" \"${module.master_instance.private_ip_address}\" \"${var.k8s_feature_gates}\"", 52 | "mkdir -p $HOME/.kube && sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && sudo chown root $HOME/.kube/config", 53 | "export PATH=$${PATH}:/opt/bin", 54 | "kubectl apply -f /root/init/calico.yaml", 55 | "chmod +x /root/init/linode-addons.sh && /root/init/linode-addons.sh \"${var.region}\" \"${var.linode_token}\"", 56 | "chmod +x /root/init/monitoring-install.sh && /root/init/monitoring-install.sh", 57 | "chmod +x /root/init/end.sh && sudo /root/init/end.sh", 58 | ] 59 | 60 | connection { 61 | user = "root" 62 | timeout = "300s" 63 | host = module.master_instance.public_ip_address 64 | } 65 | } 66 | } 67 | 68 | data "external" "kubeadm_join" { 69 | program = ["${path.cwd}/${path.module}/scripts/local/kubeadm-token.sh"] 70 | 71 | query = { 72 | host = module.master_instance.public_ip_address 73 | } 74 | 75 | depends_on = [null_resource.masters_provisioner] 76 | } 77 | -------------------------------------------------------------------------------- /modules/masters/manifests/.gitignore: -------------------------------------------------------------------------------- 1 | ccm-linode.yaml 2 | csi-linode.yaml 3 | calico.yaml 4 | -------------------------------------------------------------------------------- /modules/masters/manifests/dashboard-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: kubernetes-dashboard 5 | labels: 6 | k8s-app: kubernetes-dashboard 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: ClusterRole 10 | name: cluster-admin 11 | subjects: 12 | - kind: ServiceAccount 13 | name: kubernetes-dashboard 14 | namespace: kube-system 15 | -------------------------------------------------------------------------------- /modules/masters/manifests/dashboard.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # ------------------- Dashboard Secret ------------------- # 16 | 17 | apiVersion: v1 18 | kind: Secret 19 | metadata: 20 | labels: 21 | k8s-app: kubernetes-dashboard 22 | name: kubernetes-dashboard-certs 23 | namespace: kube-system 24 | type: Opaque 25 | 26 | --- 27 | # ------------------- Dashboard Service Account ------------------- # 28 | 29 | apiVersion: v1 30 | kind: ServiceAccount 31 | metadata: 32 | labels: 33 | k8s-app: kubernetes-dashboard 34 | name: kubernetes-dashboard 35 | namespace: kube-system 36 | 37 | --- 38 | # ------------------- Dashboard Role & Role Binding ------------------- # 39 | 40 | kind: Role 41 | apiVersion: rbac.authorization.k8s.io/v1 42 | metadata: 43 | name: kubernetes-dashboard-minimal 44 | namespace: kube-system 45 | rules: 46 | # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. 47 | - apiGroups: [""] 48 | resources: ["secrets"] 49 | verbs: ["create"] 50 | # Allow Dashboard to create 'kubernetes-dashboard-settings' config map. 51 | - apiGroups: [""] 52 | resources: ["configmaps"] 53 | verbs: ["create"] 54 | # Allow Dashboard to get, update and delete Dashboard exclusive secrets. 55 | - apiGroups: [""] 56 | resources: ["secrets"] 57 | resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] 58 | verbs: ["get", "update", "delete"] 59 | # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. 60 | - apiGroups: [""] 61 | resources: ["configmaps"] 62 | resourceNames: ["kubernetes-dashboard-settings"] 63 | verbs: ["get", "update"] 64 | # Allow Dashboard to get metrics from heapster. 65 | - apiGroups: [""] 66 | resources: ["services"] 67 | resourceNames: ["heapster"] 68 | verbs: ["proxy"] 69 | - apiGroups: [""] 70 | resources: ["services/proxy"] 71 | resourceNames: ["heapster", "http:heapster:", "https:heapster:"] 72 | verbs: ["get"] 73 | 74 | --- 75 | apiVersion: rbac.authorization.k8s.io/v1 76 | kind: RoleBinding 77 | metadata: 78 | name: kubernetes-dashboard-minimal 79 | namespace: kube-system 80 | roleRef: 81 | apiGroup: rbac.authorization.k8s.io 82 | kind: Role 83 | name: kubernetes-dashboard-minimal 84 | subjects: 85 | - kind: ServiceAccount 86 | name: kubernetes-dashboard 87 | namespace: kube-system 88 | 89 | --- 90 | # ------------------- Dashboard Deployment ------------------- # 91 | 92 | kind: Deployment 93 | apiVersion: apps/v1 94 | metadata: 95 | labels: 96 | k8s-app: kubernetes-dashboard 97 | name: kubernetes-dashboard 98 | namespace: kube-system 99 | spec: 100 | replicas: 1 101 | revisionHistoryLimit: 10 102 | selector: 103 | matchLabels: 104 | k8s-app: kubernetes-dashboard 105 | template: 106 | metadata: 107 | labels: 108 | k8s-app: kubernetes-dashboard 109 | spec: 110 | containers: 111 | - name: kubernetes-dashboard 112 | image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 113 | ports: 114 | - containerPort: 8443 115 | protocol: TCP 116 | args: 117 | - --auto-generate-certificates 118 | # Uncomment the following line to manually specify Kubernetes API server Host 119 | # If not specified, Dashboard will attempt to auto discover the API server and connect 120 | # to it. Uncomment only if the default does not work. 121 | # - --apiserver-host=http://my-address:port 122 | volumeMounts: 123 | - name: kubernetes-dashboard-certs 124 | mountPath: /certs 125 | # Create on-disk volume to store exec logs 126 | - mountPath: /tmp 127 | name: tmp-volume 128 | livenessProbe: 129 | httpGet: 130 | scheme: HTTPS 131 | path: / 132 | port: 8443 133 | initialDelaySeconds: 30 134 | timeoutSeconds: 30 135 | volumes: 136 | - name: kubernetes-dashboard-certs 137 | secret: 138 | secretName: kubernetes-dashboard-certs 139 | - name: tmp-volume 140 | emptyDir: {} 141 | serviceAccountName: kubernetes-dashboard 142 | # Comment the following tolerations if Dashboard must not be deployed on master 143 | tolerations: 144 | - key: node-role.kubernetes.io/master 145 | effect: NoSchedule 146 | 147 | --- 148 | # ------------------- Dashboard Service ------------------- # 149 | 150 | kind: Service 151 | apiVersion: v1 152 | metadata: 153 | labels: 154 | k8s-app: kubernetes-dashboard 155 | name: kubernetes-dashboard 156 | namespace: kube-system 157 | spec: 158 | ports: 159 | - port: 443 160 | targetPort: 8443 161 | selector: 162 | k8s-app: kubernetes-dashboard -------------------------------------------------------------------------------- /modules/masters/manifests/external-dns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: external-dns 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | kind: ClusterRole 9 | metadata: 10 | name: external-dns 11 | namespace: kube-system 12 | rules: 13 | - apiGroups: [""] 14 | resources: ["services"] 15 | verbs: ["get","watch","list"] 16 | - apiGroups: [""] 17 | resources: ["pods"] 18 | verbs: ["get","watch","list"] 19 | - apiGroups: ["extensions"] 20 | resources: ["ingresses"] 21 | verbs: ["get","watch","list"] 22 | - apiGroups: [""] 23 | resources: ["nodes"] 24 | verbs: ["list"] 25 | --- 26 | apiVersion: rbac.authorization.k8s.io/v1beta1 27 | kind: ClusterRoleBinding 28 | metadata: 29 | name: external-dns-viewer 30 | namespace: kube-system 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: external-dns 35 | subjects: 36 | - kind: ServiceAccount 37 | name: external-dns 38 | namespace: kube-system 39 | --- 40 | apiVersion: apps/v1 41 | kind: Deployment 42 | metadata: 43 | name: external-dns 44 | namespace: kube-system 45 | spec: 46 | selector: 47 | matchLabels: 48 | app: external-dns 49 | strategy: 50 | type: Recreate 51 | template: 52 | metadata: 53 | labels: 54 | app: external-dns 55 | spec: 56 | serviceAccountName: external-dns 57 | containers: 58 | - name: external-dns 59 | image: registry.opensource.zalan.do/teapot/external-dns:latest 60 | args: 61 | - --provider=linode 62 | # - --domain-filter=example.com # (optional) limit to only example.com domains; change to match the zone created above. 63 | env: 64 | - name: EXTERNAL_DNS_SOURCE 65 | value: |- 66 | service 67 | ingress 68 | - name: LINODE_TOKEN 69 | valueFrom: 70 | secretKeyRef: 71 | name: linode 72 | key: token 73 | -------------------------------------------------------------------------------- /modules/masters/manifests/linode-token.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: linode 5 | namespace: kube-system 6 | stringData: 7 | token: "$(LINODE_TOKEN)" 8 | region: "$(LINODE_REGION)" 9 | -------------------------------------------------------------------------------- /modules/masters/manifests/metrics-server.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ClusterRole 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: system:aggregated-metrics-reader 6 | labels: 7 | rbac.authorization.k8s.io/aggregate-to-view: "true" 8 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 9 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 10 | rules: 11 | - apiGroups: ["metrics.k8s.io"] 12 | resources: ["pods"] 13 | verbs: ["get", "list", "watch"] 14 | --- 15 | apiVersion: rbac.authorization.k8s.io/v1beta1 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: metrics-server:system:auth-delegator 19 | roleRef: 20 | apiGroup: rbac.authorization.k8s.io 21 | kind: ClusterRole 22 | name: system:auth-delegator 23 | subjects: 24 | - kind: ServiceAccount 25 | name: metrics-server 26 | namespace: kube-system 27 | --- 28 | apiVersion: rbac.authorization.k8s.io/v1beta1 29 | kind: RoleBinding 30 | metadata: 31 | name: metrics-server-auth-reader 32 | namespace: kube-system 33 | roleRef: 34 | apiGroup: rbac.authorization.k8s.io 35 | kind: Role 36 | name: extension-apiserver-authentication-reader 37 | subjects: 38 | - kind: ServiceAccount 39 | name: metrics-server 40 | namespace: kube-system 41 | --- 42 | apiVersion: apiregistration.k8s.io/v1beta1 43 | kind: APIService 44 | metadata: 45 | name: v1beta1.metrics.k8s.io 46 | spec: 47 | service: 48 | name: metrics-server 49 | namespace: kube-system 50 | group: metrics.k8s.io 51 | version: v1beta1 52 | insecureSkipTLSVerify: true 53 | groupPriorityMinimum: 100 54 | versionPriority: 100 55 | --- 56 | apiVersion: v1 57 | kind: ServiceAccount 58 | metadata: 59 | name: metrics-server 60 | namespace: kube-system 61 | --- 62 | apiVersion: apps/v1 63 | kind: Deployment 64 | metadata: 65 | name: metrics-server 66 | namespace: kube-system 67 | labels: 68 | k8s-app: metrics-server 69 | spec: 70 | selector: 71 | matchLabels: 72 | k8s-app: metrics-server 73 | template: 74 | metadata: 75 | name: metrics-server 76 | labels: 77 | k8s-app: metrics-server 78 | spec: 79 | serviceAccountName: metrics-server 80 | volumes: 81 | # mount in tmp so we can safely use from-scratch images and/or read-only containers 82 | - name: tmp-dir 83 | emptyDir: {} 84 | containers: 85 | - name: metrics-server 86 | image: k8s.gcr.io/metrics-server-amd64:v0.3.1 87 | imagePullPolicy: Always 88 | command: 89 | - /metrics-server 90 | - --kubelet-insecure-tls 91 | - --kubelet-preferred-address-types=InternalIP 92 | volumeMounts: 93 | - name: tmp-dir 94 | mountPath: /tmp 95 | --- 96 | apiVersion: v1 97 | kind: Service 98 | metadata: 99 | name: metrics-server 100 | namespace: kube-system 101 | labels: 102 | kubernetes.io/name: "Metrics-server" 103 | spec: 104 | selector: 105 | k8s-app: metrics-server 106 | ports: 107 | - port: 443 108 | protocol: TCP 109 | targetPort: 443 110 | --- 111 | apiVersion: rbac.authorization.k8s.io/v1 112 | kind: ClusterRole 113 | metadata: 114 | name: system:metrics-server 115 | rules: 116 | - apiGroups: 117 | - "" 118 | resources: 119 | - pods 120 | - nodes 121 | - nodes/stats 122 | - namespaces 123 | verbs: 124 | - get 125 | - list 126 | - watch 127 | - apiGroups: 128 | - "extensions" 129 | resources: 130 | - deployments 131 | verbs: 132 | - get 133 | - list 134 | - watch 135 | --- 136 | apiVersion: rbac.authorization.k8s.io/v1 137 | kind: ClusterRoleBinding 138 | metadata: 139 | name: system:metrics-server 140 | roleRef: 141 | apiGroup: rbac.authorization.k8s.io 142 | kind: ClusterRole 143 | name: system:metrics-server 144 | subjects: 145 | - kind: ServiceAccount 146 | name: metrics-server 147 | namespace: kube-system -------------------------------------------------------------------------------- /modules/masters/outputs.tf: -------------------------------------------------------------------------------- 1 | output "k8s_master_public_ip" { 2 | depends_on = [module.master_instance] 3 | description = "Public IP Address of the master node" 4 | value = module.master_instance.public_ip_address 5 | } 6 | 7 | output "k8s_master_private_ip" { 8 | depends_on = [module.master_instance] 9 | description = "Private IP Address of the master node" 10 | value = module.master_instance.private_ip_address 11 | } 12 | 13 | locals { 14 | result = { 15 | command = "" 16 | } 17 | 18 | kubeadm_join_results = concat(data.external.kubeadm_join.*.result, [local.result]) 19 | kubeadm_join_command = lookup(local.kubeadm_join_results["0"], "command", "") 20 | } 21 | 22 | output "kubeadm_join_command" { 23 | depends_on = [null_resource.masters_provisioner] 24 | description = "kubeadm join command that can be used to add a node to the cluster" 25 | value = local.kubeadm_join_command 26 | } 27 | -------------------------------------------------------------------------------- /modules/masters/scripts/local/kubeadm-token.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # Extract "host" argument from the input into HOST shell variable 6 | eval "$(python3 -c 'import sys, json; print("HOST="+json.load(sys.stdin)["host"])')" 2>/dev/null || true 7 | 8 | # TODO: pass the ssh key into this command 9 | # Fetch the join command 10 | if [ -z "$HOST" ]; then 11 | echo "{\"command\":\"\"}" 12 | else 13 | CMD=$(ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ 14 | root@$HOST sudo kubeadm token create --print-join-command | awk '/\\$/ { printf "%s", substr($0, 1, length($0)-1); next }; /kubeadm/ && ! /control-plane/ {gsub(/^[ \t]/, "", $0); print $0}' ) 15 | # Produce a JSON object containing the join command 16 | CMD="$CMD --discovery-token-unsafe-skip-ca-verification" 17 | echo '{"command":"'"${CMD}"'"}' 18 | fi 19 | -------------------------------------------------------------------------------- /modules/masters/templates/ccm-linode.yaml.template: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: ccm-linode 5 | namespace: kube-system 6 | --- 7 | kind: ClusterRoleBinding 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: system:ccm-linode 11 | roleRef: 12 | apiGroup: rbac.authorization.k8s.io 13 | kind: ClusterRole 14 | # TODO: make these permissions more fine-grained 15 | name: cluster-admin 16 | subjects: 17 | - kind: ServiceAccount 18 | name: ccm-linode 19 | namespace: kube-system 20 | --- 21 | apiVersion: apps/v1 22 | kind: DaemonSet 23 | metadata: 24 | name: ccm-linode 25 | labels: 26 | app: ccm-linode 27 | namespace: kube-system 28 | spec: 29 | selector: 30 | matchLabels: 31 | app: ccm-linode 32 | template: 33 | metadata: 34 | labels: 35 | app: ccm-linode 36 | spec: 37 | serviceAccountName: ccm-linode 38 | nodeSelector: 39 | # The CCM will only run on a Node labelled as a master, you may want to change this 40 | node-role.kubernetes.io/master: "" 41 | tolerations: 42 | # The CCM can run on Nodes tainted as masters 43 | - key: "node-role.kubernetes.io/master" 44 | effect: "NoSchedule" 45 | # The CCM is a "critical addon" 46 | - key: "CriticalAddonsOnly" 47 | operator: "Exists" 48 | # This taint is set on all Nodes when an external CCM is used 49 | - key: node.cloudprovider.kubernetes.io/uninitialized 50 | value: "true" 51 | effect: NoSchedule 52 | - key: node.kubernetes.io/not-ready 53 | operator: Exists 54 | effect: NoSchedule 55 | - key: node.kubernetes.io/unreachable 56 | operator: Exists 57 | effect: NoSchedule 58 | hostNetwork: true 59 | containers: 60 | - image: {{ .Values.CCMImage }} 61 | imagePullPolicy: Always 62 | name: ccm-linode 63 | args: 64 | - --cloud-provider=linode 65 | - --leader-elect-resource-lock=endpoints 66 | - --v=3 67 | volumeMounts: 68 | - mountPath: /etc/kubernetes 69 | name: k8s 70 | env: 71 | - name: LINODE_API_TOKEN 72 | valueFrom: 73 | secretKeyRef: 74 | name: linode 75 | key: token 76 | - name: LINODE_REGION 77 | valueFrom: 78 | secretKeyRef: 79 | name: linode 80 | key: region 81 | volumes: 82 | - name: k8s 83 | hostPath: 84 | path: /etc/kubernetes 85 | -------------------------------------------------------------------------------- /modules/masters/variables.tf: -------------------------------------------------------------------------------- 1 | variable "ubuntu_version" { 2 | description = "Ubuntu version to install" 3 | default = "20.04" 4 | } 5 | 6 | variable "node_count" { 7 | default = "1" 8 | description = "Number of Kubernetes Control-Plane nodes to provision (max 1)" 9 | } 10 | 11 | variable "node_class" { 12 | default = "node" 13 | description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)" 14 | } 15 | 16 | variable "node_type" { 17 | default = "g6-standard-4" 18 | description = "Linode Instance type for nodes" 19 | } 20 | 21 | variable "private_ip" { 22 | default = true 23 | description = "Enables Linode Instance Private IP addresses" 24 | } 25 | 26 | variable "label_prefix" { 27 | default = "" 28 | description = "Linode label prefix" 29 | } 30 | 31 | variable "linode_token" { 32 | description = "Linode Token used for CSI, CCM, an other Linode addons in the cluster" 33 | } 34 | 35 | variable "k8s_version" { 36 | description = "Kubernetes version to install" 37 | } 38 | 39 | variable "cni_version" { 40 | description = "Container Network Plugin Version" 41 | } 42 | 43 | variable "crictl_version" { 44 | description = "Container Runtime Interface version to install" 45 | } 46 | 47 | variable "cluster_name" { 48 | description = "Name of the Kubernetes cluster" 49 | } 50 | 51 | variable "ssh_public_key" { 52 | description = "SSH keys authorized for the root user account" 53 | } 54 | 55 | variable "linode_group" { 56 | default = "" 57 | description = "Linode display group for provisioned instances" 58 | } 59 | 60 | variable "k8s_feature_gates" { 61 | description = "Feature gates to enable in the Kubelet and API server" 62 | } 63 | 64 | variable "region" { 65 | description = "Linode Region: us-central us-west us-southeast us-east eu-west ap-south eu-central ap-northeast ap-northeast-1a" 66 | } 67 | 68 | variable "docker_version" { 69 | description = "The docker version to install" 70 | } 71 | -------------------------------------------------------------------------------- /modules/nodes/README.md: -------------------------------------------------------------------------------- 1 | ## Ubuntu Kubernetes Nodes Instances 2 | 3 | This module provisions a Kubernetes Node Linode Instance using Ubuntu. 4 | -------------------------------------------------------------------------------- /modules/nodes/main.tf: -------------------------------------------------------------------------------- 1 | module "node" { 2 | source = "../instances" 3 | label_prefix = var.label_prefix 4 | node_type = var.node_type 5 | node_count = var.node_count 6 | node_class = "node" 7 | private_ip = "true" 8 | ssh_public_key = var.ssh_public_key 9 | region = var.region 10 | 11 | linode_group = var.linode_group 12 | 13 | ubuntu_version = var.ubuntu_version 14 | k8s_version = var.k8s_version 15 | k8s_feature_gates = var.k8s_feature_gates 16 | cni_version = var.cni_version 17 | crictl_version = var.crictl_version 18 | docker_version = var.docker_version 19 | } 20 | 21 | // todo: does the use of var.kubeadm_join_command (from master output) queue nodes behind masters? move to parent main.tf if so 22 | resource "null_resource" "kubeadm_join" { 23 | count = var.node_count 24 | 25 | provisioner "remote-exec" { 26 | inline = [ 27 | "set -e", 28 | "export PATH=$${PATH}:/opt/bin", 29 | "sudo ${var.kubeadm_join_command}", 30 | "chmod +x /root/init/end.sh && sudo /root/init/end.sh", 31 | ] 32 | 33 | connection { 34 | host = element(module.node.nodes_public_ip, count.index) 35 | user = "root" 36 | timeout = "300s" 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /modules/nodes/outputs.tf: -------------------------------------------------------------------------------- 1 | output "nodes_public_ip" { 2 | description = "Public IP Address of the worker nodes" 3 | value = module.node.nodes_public_ip 4 | } 5 | -------------------------------------------------------------------------------- /modules/nodes/variables.tf: -------------------------------------------------------------------------------- 1 | variable "ubuntu_version" { 2 | description = "Ubuntu version to install" 3 | default = "20.04" 4 | } 5 | 6 | variable "node_count" { 7 | default = "1" 8 | description = "Number of Kubernetes Nodes to provision" 9 | } 10 | 11 | variable "node_class" { 12 | default = "node" 13 | description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)" 14 | } 15 | 16 | variable "node_type" { 17 | default = "g6-standard-4" 18 | description = "Linode Instance type for nodes" 19 | } 20 | 21 | variable "private_ip" { 22 | default = true 23 | description = "Enables Linode Instance Private IP addresses" 24 | } 25 | 26 | variable "label_prefix" { 27 | default = "" 28 | description = "Linode label prefix" 29 | } 30 | 31 | variable "linode_group" { 32 | default = "" 33 | description = "Linode display group for provisioned instances" 34 | } 35 | 36 | variable "kubeadm_join_command" { 37 | description = "Kubernetes 'kubeadm join' command to join this node to the cluster" 38 | } 39 | 40 | variable "region" { 41 | description = "Linode region for instances" 42 | } 43 | 44 | variable "ssh_public_key" { 45 | description = "SSH keys authorized for the root user account" 46 | } 47 | 48 | variable "k8s_version" { 49 | description = "Kubernetes version to install" 50 | } 51 | 52 | variable "cni_version" { 53 | description = "CNI version to install" 54 | } 55 | 56 | variable "crictl_version" { 57 | description = "Container Runtime Interface version to install" 58 | } 59 | 60 | variable "k8s_feature_gates" { 61 | description = "Kubernetes Feature gates to enable in the Kubelet" 62 | } 63 | 64 | variable "docker_version" { 65 | description = "The docker version to install" 66 | } 67 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | output "k8s_master_public_ip" { 2 | description = "Public IP Address of the Kubernetes API Server" 3 | value = module.masters.k8s_master_public_ip 4 | } 5 | 6 | output "kubeadm_join_command" { 7 | description = "kubeadm join command that can be used to add a node to the cluster" 8 | value = module.masters.kubeadm_join_command 9 | sensitive = true 10 | } 11 | 12 | output "nodes_public_ip" { 13 | description = "Public IP Address of the worker nodes" 14 | value = module.nodes.nodes_public_ip 15 | } 16 | 17 | output "kubectl_config" { 18 | description = "Filename of the Kubernetes config file" 19 | value = "${terraform.workspace}.conf" 20 | } 21 | -------------------------------------------------------------------------------- /screens/dash-nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linode/terraform-linode-k8s/efc69bb2a30a0a4982cb721e8ac40b366a61d29c/screens/dash-nodes.png -------------------------------------------------------------------------------- /screens/dash-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linode/terraform-linode-k8s/efc69bb2a30a0a4982cb721e8ac40b366a61d29c/screens/dash-overview.png -------------------------------------------------------------------------------- /scripts/local/kubectl-conf.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | WORKSPACE=$1 5 | PUBLIC_IP=$2 6 | PRIVATE_IP=$3 7 | SSH_KEY=$4 8 | 9 | scp -i ${SSH_KEY} -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${PUBLIC_IP}:/root/.kube/config admin.conf 10 | sed -e "s/${PRIVATE_IP}/${PUBLIC_IP}/g" admin.conf > ${WORKSPACE}.conf 11 | rm admin.conf 12 | -------------------------------------------------------------------------------- /scripts/local/preflight.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euf -o pipefail 3 | 4 | function assertInstalled() { 5 | for var in "$@"; do 6 | if ! which $var &> /dev/null; then 7 | echo "$var not found" 8 | exit 1 9 | fi 10 | done 11 | } 12 | 13 | assertInstalled ssh scp sed kubectl python 14 | 15 | CCM_IMAGE=$1 16 | CSI_MANIFEST=$2 17 | CALICO_MANIFEST=$3 18 | 19 | MANIFESTS_DIR=./modules/masters/manifests 20 | 21 | # substitute Docker images in manifests 22 | 23 | sed -e "s|{{ \.Values\.CCMImage }}|${CCM_IMAGE}|g" modules/masters/templates/ccm-linode.yaml.template > $MANIFESTS_DIR/ccm-linode.yaml 24 | curl "${CSI_MANIFEST}" -o $MANIFESTS_DIR/csi-linode.yaml 25 | curl "${CALICO_MANIFEST}" -o $MANIFESTS_DIR/calico.yaml 26 | -------------------------------------------------------------------------------- /terraform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export TF_SCHEMA_PANIC_ON_ERROR=1 4 | export TF_LOG=TRACE 5 | rm `pwd`/linode-test.log || true 6 | export TF_LOG_PATH=`pwd`/linode-test.log 7 | export LINODE_DEBUG=1 8 | 9 | cmd=apply 10 | if [ -n "$1" ]; then 11 | cmd="$1" 12 | shift 13 | fi 14 | terraform "$cmd" $@ 15 | -------------------------------------------------------------------------------- /terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.0.0" 3 | } 4 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "ubuntu_version" { 2 | description = "Ubuntu version to install" 3 | default = "20.04" 4 | } 5 | 6 | variable "cni_version" { 7 | default = "v0.8.7" 8 | description = "Container Network Plugin Version" 9 | } 10 | 11 | variable "k8s_version" { 12 | default = "v1.18.13" 13 | description = "Kubernetes version to install" 14 | } 15 | 16 | variable "crictl_version" { 17 | default = "v1.15.0" 18 | description = "Container Runtime Interface version to install" 19 | } 20 | 21 | variable "k8s_feature_gates" { 22 | default = "" 23 | description = "Feature gates to enable in the Kubelet and API server" 24 | } 25 | 26 | variable "region" { 27 | default = "eu-west" 28 | description = "Linode Region: us-central us-west us-southeast us-east eu-west ap-south eu-central ap-northeast ap-northeast-1a" 29 | } 30 | 31 | variable "server_type_master" { 32 | default = "g6-standard-2" 33 | description = "Linode Instance type: g6-nanode-1 g6-standard-1 g6-standard-2 g6-standard-4 g6-standard-6 g6-standard-8 g6-standard-16 g6-standard-20 g6-standard-24 g6-standard-32 g6-highmem-1 g6-highmem-2 g6-highmem-4 g6-highmem-8 g6-highmem-16" 34 | } 35 | 36 | variable "server_type_node" { 37 | default = "g6-standard-2" 38 | description = "Linode Instance type: g6-nanode-1 g6-standard-1 g6-standard-2 g6-standard-4 g6-standard-6 g6-standard-8 g6-standard-16 g6-standard-20 g6-standard-24 g6-standard-32 g6-highmem-1 g6-highmem-2 g6-highmem-4 g6-highmem-8 g6-highmem-16" 39 | } 40 | 41 | variable "masters" { 42 | default = 1 43 | description = "Number of control-plane (master) nodes. This must be 1 for now." 44 | } 45 | 46 | variable "nodes" { 47 | default = 3 48 | description = "Number of worker nodes to provision" 49 | } 50 | 51 | variable "cluster_name" { 52 | default = "" 53 | description = "Name of the Kubernetes cluster" 54 | } 55 | 56 | variable "linode_token" { 57 | type = string 58 | description = "Linode API v4 Personal Access Token" 59 | } 60 | 61 | variable "ssh_public_key" { 62 | type = string 63 | default = "~/.ssh/id_rsa.pub" 64 | description = "The path to your public key" 65 | } 66 | 67 | variable "ccm_image" { 68 | type = string 69 | default = "linode/linode-cloud-controller-manager:latest" 70 | description = "The docker repo/image:tag to use for the CCM" 71 | } 72 | 73 | variable "csi_manifest" { 74 | type = string 75 | default = "https://raw.githubusercontent.com/linode/linode-blockstorage-csi-driver/master/pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.7.yaml" 76 | description = "The linode csi manifest location" 77 | } 78 | 79 | variable "calico_manifest" { 80 | type = string 81 | default = "https://docs.projectcalico.org/manifests/calico.yaml" 82 | description = "The calico manifest location" 83 | } 84 | 85 | variable "docker_version" { 86 | type = string 87 | default = "19.03" 88 | description = "The docker version to install" 89 | } 90 | -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | external = { 4 | source = "hashicorp/external" 5 | version = "~> 2.0.0" 6 | } 7 | linode = { 8 | source = "linode/linode" 9 | version = "~> 1.16.0" 10 | } 11 | null = { 12 | source = "hashicorp/null" 13 | version = "~> 3.0.0" 14 | } 15 | } 16 | } 17 | --------------------------------------------------------------------------------