├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md └── aws ├── cluster ├── cluster-head-nodes-template.tf ├── cluster-spec-template.tf ├── data.tf ├── main.tf ├── networking.tf ├── outputs.tf ├── route53.tf ├── templates │ ├── cluster-spec.yaml │ └── ig-spec.yaml └── variables.tf ├── ig ├── data.tf ├── ig-spec-template.tf ├── main.tf ├── outputs.tf ├── templates │ └── spec.yaml └── variables.tf └── test ├── data.tf ├── dns.tf ├── iam-kops.tf ├── k8s ├── backend │ └── pong.yaml ├── ingress │ ├── default-backend.yaml │ ├── kube-lego.yaml │ ├── nginx-ingress-controller-config.yaml │ └── nginx-ingress-controller.yaml ├── kube-system │ ├── cluster-autoscaler.yaml │ ├── critical-addons-rescheduler.yaml │ ├── dashboard.yaml │ ├── heapster.yaml │ ├── kube-dns.yaml │ ├── kube-state-metrics.yaml │ └── npd.yaml └── namespaces.yaml ├── provider.tf ├── test-cluster.tf ├── test-cluster ├── ingress.tf ├── kernel-tuning.tf ├── kops-cluster.tf ├── outputs.tf ├── provider.tf └── variables.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | ## Terraform 2 | # Compiled files 3 | *.tfstate 4 | *.tfstate.backup 5 | 6 | # Module directory 7 | .terraform/ 8 | 9 | ## Vim 10 | # Swap 11 | [._]*.s[a-v][a-z] 12 | [._]*.sw[a-p] 13 | [._]s[a-v][a-z] 14 | [._]sw[a-p] 15 | 16 | # Session 17 | Session.vim 18 | 19 | # Temporary 20 | .netrwhist 21 | *~ 22 | # Auto-generated tag files 23 | tags 24 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | --------- 3 | 4 | ### 1.7.1 5 | 6 | **Note**: from this release, `karch` minor releases (ex.: 1.8.x) will be made 7 | against the corresponding `kops` release. On CoreOS, compatibility is guaranteed 8 | between these versions. You should therefore use `karch` `v1.8.x` with `kops` 9 | `v1.8.y`. 10 | 11 | #### Added 12 | * It is now possible to configure (runtime,kubelet) and system resource 13 | reservation on nodes 14 | * The APIServer `--runtime-config` flag is now exposed, allowing users to 15 | easily enable/disable Kubernetes API groups 16 | * `batch/v2alpha1` 17 | 18 | #### Changed 19 | * [refactoring] Remove efs/ebs-pv and revisit folder structure 20 | 21 | Coupling the creation of a complete Kubernetes cluster and of resources 22 | (PersistentVolumes, in the case of the efs-pv and ebs-fv volumes) was a 23 | really bad idea: 24 | * If the Kubernetes API Server is down, it blocks any plan/apply on 25 | your Terraform code using these modules, being unable to run 26 | Terraform when an outage occured in a cluster isn't... cool :( 27 | * It even prevents you to create a custom cluster from scratch 28 | 29 | It is therefore advised to create your Kubernetes infrastructure 30 | (including EBS/EFS volumes, karch module calls...) in one piece of 31 | Terraform code and to use the "terraform_remote_state" datasources from 32 | another to retrieve some data you'd need to provision resources (such 33 | as Secrets or PersistentVolumes) from the original codebase or - more 34 | precisely - from the state bound to it. 35 | * Some alpha API Groups have been enabled. This will soon be configurable with 36 | a list variable 37 | 38 | #### Removed 39 | * `ebs-pv` and `efs-pv` volumes: these were a really bad idea: they imply that, 40 | for each `plan` or `apply` call you'll be willing to make, the Kubernetes API 41 | server must be up. This is totally impossible during cluster bootstrap 42 | (without commenting code) and when the API server is unreachable. You don't 43 | want your `terraform apply` to be down during outages :) 44 | 45 | ### 0.4.1 46 | 47 | #### Fixed 48 | * Relative path handling issue wrt Terraform 10.0.6 49 | * OIDC Server templating fix 50 | 51 | #### Changed 52 | * Use dashes in all output names 53 | 54 | ### 0.4.0 55 | ode security group ID as output of the `kops-cluster` module 56 | * Helper module to provision EBS volumes as kubernetes persistent volumes 57 | (includes the ability to restore them from a snapshot) 58 | * Possibility to configure an OIDC provider to the Kubernetes API (tested with 59 | Google and Github) 60 | 61 | #### Changed 62 | * We're not creating an S3 VPC endpoint automatically, however you can create 63 | it outside of the `kops-cluster` module using the exported `vpc-id`. 64 | 65 | ### 0.3.0 - RBAC ! 66 | 67 | #### Added 68 | * It is now possible to create `kops` clusters with RBAC enabled by default 69 | * etcd volumes are encrypted 70 | * support for kops ability to disable the automatic creation of security groups 71 | when creating a LoadBalancer service 72 | 73 | #### Fixed 74 | * During `terraform destroy`, avoid errors that occur in case the cluster has 75 | already been deleted outside of Terraform 76 | * Kops cluster domain and your cluster's kube-dns domain are now setup 77 | accordingly 78 | 79 | ### 0.2.0 - Add relevant variables and outputs to our kops modules 80 | 81 | #### Fixed 82 | * premature Internet Gateway destroy triggering Terraform errors on destroyed 83 | (could be worked around by running destroy twice) 84 | 85 | #### Added 86 | * the ASG generated by kops with the `kops-ig` module now have their names 87 | exposed as outputs (so that you can - for instance - expose your ingress 88 | controllers as a NodePort on the host and wire them to an ELB) 89 | * Terraform variable to attach custom security groups to an instance group 90 | (for the same purpose essentially) 91 | * kops-created subnets now have their subnet ids exposed as outputs (so that 92 | you can use Terraform to create resources in these same subnets) 93 | 94 | ### 0.1.0 - First release of Karch 95 | 96 | A Terraform module designed as a wrapper around kops, it makes it possible to 97 | spawn kops clusters using the `kops-cluster` and `kops-ip` modules, supports 98 | cluster updates and deletes and exposes resources created by kops to the rest of 99 | your Terraform code, as simple Terraform outputs. 100 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | karch - A terraform module to spawn Kubernetes clusters 2 | ======================================================= 3 | 4 | `karch` is a Terraform module based on 5 | [kops](https://github.com/kubernetes/kops) aiming at managing (multiple) 6 | Kubernetes clusters on AWS. You can see it as "Terraform bindings for kops". 7 | 8 | It essentially aims at making it easier to share Kubernetes cluster topologies 9 | or even entire stacks built atop Kubernetes. 10 | 11 | Motivations 12 | ----------- 13 | `kops` has become the standard, non-opinionated way of deploying Kubernetes 14 | clusters on AWS and can even generate Terraform code. However, this approach has 15 | some limits: 16 | * Values of resources managed by `kops`, such as the id of the cluster's VPC, 17 | subnets, etc... aren't really accessible from the rest of your codebase. 18 | * One needs one subfolder per cluster (which can be used as a Terraform 19 | module): creating a "cluster template" (masters + several IGs) that can 20 | easily be replicated accross AWS regions & shared accross teams isn't 21 | possible 22 | 23 | It seemed that wrapping by wrapping the `kops` CLI itself into a Terraform 24 | module whicch really feels like a simple Terraform module could fulfill this 25 | need for portable, reapeatable infrastructure a bit better. Of course, keeping 26 | the flexibilty offered by `kops`'s cluster & instance group spec available by 27 | exposing all the parameters it provides as Terraform variables felt essential. 28 | 29 | Therfore, `karch` aims at making it easy to encode Kubernetes cluster topologies 30 | using Terraform infrastructure code. For instance, such a topology could be: 31 | - an instance group for a pool of NginX ingress controllers, mounting ports 32 | - one for your backend APIs 33 | - one for stateful apps (databases, data stores...) 34 | - one, with GPU instances, to run your ML pipeline 35 | - with Kubernetes to orchestrate all types of workloads 36 | 37 | What `karch` is 38 | --------------- 39 | * A Terraform library, written in plain HCL and using essentially `kops`, `sh` 40 | and `awk`. 41 | * A set of two Terraform modules `cluster` and `ig`. The former spaws a base 42 | cluster, in a new VPC, the latter can be used to spawn instance groups. 43 | * A wrapper around `kops`, instead of using `kops` directly, you'll be using 44 | a terraform module to create/update/delete your `kops` clusters. When 45 | necessary, this module will take care of rolling out your instance groups. 46 | 47 | What `karch` isn't 48 | ------------------ 49 | * A Terraform provider **plugin**. Writing such a plugin would be nice, but 50 | would require much more time to implement. 51 | * For now, `karch` spawns only clusters with a `private` topology. Adding the 52 | ability to create `public` clusters will come next 53 | * For now, `karch` takes care of creating a VPC and Route53 zone for your 54 | cluster's subdomain. Being able to give it an already existing VPC and/or 55 | zone is on the roadmap 56 | 57 | Getting started 58 | --------------- 59 | #### Requirements 60 | You'll only need `kops`, `kubectl`, `sh`, and the `aws-cli` (or at 61 | least, an AWS account configured `accordingly` under `~/.aws/credentials`). 62 | 63 | #### Creating a Kubernetes cluster 64 | 65 | To create a Kubernetes cluster, you can use the `kops-cluster` module: 66 | You can refer to `./kops-cluster/variables.tf` for a documented list of all the 67 | variables you can pass to the module. 68 | ``` 69 | module "kops-cluster" { 70 | source = "github.com/elafarge/karch/aws/cluster" 71 | version = "1.7.1" 72 | 73 | aws-region = "eu-west-1" 74 | 75 | # Networking & connectivity 76 | vpc-name = "kube-hq" 77 | vpc-cidr = "10.70.0.0/16" 78 | availability-zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 79 | kops-topology = "private" 80 | trusted-cidrs = "0.0.0.0/0" 81 | admin-ssh-public-key-path = "~/.ssh/id_rsa.pub" 82 | 83 | # DNS 84 | main-zone-id = "example.com" 85 | cluster-name = "kube-hq.example.com" 86 | 87 | # Kops & Kuberntetes 88 | kops-state-bucket = "example-com-kops-state" 89 | 90 | # Master 91 | master-availability-zones = ["eu-west-1a"] 92 | master-image = "ami-109d6069" 93 | 94 | # Bastion 95 | bastion-image = "ami-109d6069" 96 | 97 | # First minion instance group 98 | minion-image = "ami-109d6069" 99 | } 100 | ``` 101 | 102 | #### Adding instance groups to the cluster 103 | 104 | Here as well, it boils down to simply using a Terraform module. The list of 105 | accepted variables can be found under `./kops-ig/variables.tf`. 106 | ``` 107 | module "ingress-ig" { 108 | source = "github.com/elafarge/karch/aws/ig" 109 | version = "1.7.1" 110 | 111 | aws-region = "eu-west-1" 112 | 113 | # Master cluster dependency hook 114 | master-up = "${module.kops-cluster.master-up}" 115 | 116 | # Global config 117 | name = "ingress" 118 | cluster-name = "kube-hq.example.com" 119 | kops-state-bucket = "example-com-kops-state" 120 | visibility = "private" 121 | subnets = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 122 | image = "ami-109d6069" 123 | type = "t2.small" 124 | volume-size = "16" 125 | volume-type = "gp2" 126 | min-size = 2 127 | max-size = 3 128 | node-labels = "${map("role.node", "ingress")}" 129 | } 130 | ``` 131 | 132 | #### Mainting your cluster 133 | You can entirely rely on Terraform to update your cluster on `terraform apply`. 134 | Please note that we never run `kops rolling-update` for cluster updates. You'll 135 | need to run it manually. However, rolling updates can be automatically applied 136 | for instance groups, with a configurable node rollout time interval. 137 | 138 | Maintainers 139 | ----------- 140 | * Étienne Lafarge 141 | -------------------------------------------------------------------------------- /aws/cluster/cluster-head-nodes-template.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "master-spec" { 2 | count = length(var.master-availability-zones) 3 | template = file("${path.module}/templates/ig-spec.yaml") 4 | 5 | vars = { 6 | cluster-name = var.cluster-name 7 | cloud-labels = join("\n", data.template_file.master-cloud-labels.*.rendered) 8 | node-labels = join("\n", data.template_file.master-node-labels.*.rendered) 9 | name = "master-${element(var.master-availability-zones, count.index)}" 10 | public = "false" 11 | 12 | additional-sgs = < 0 ? "additionalSecurityGroups:" : ""} 14 | ${join("\n", data.template_file.master-additional-sgs.*.rendered)} 15 | EOF 16 | 17 | image = var.master-image 18 | type = var.master-machine-type 19 | max-size = 1 20 | min-size = 1 21 | role = "Master" 22 | volume-size = var.master-volume-size 23 | volume-provisioned-iops = var.master-volume-provisioned-iops == "" ? "" : var.master-volume-provisioned-iops 24 | volume-type = var.master-volume-type 25 | ebs-optimized = var.master-ebs-optimized 26 | max-price = "" 27 | taints = "" 28 | subnets = " - ${element(var.master-availability-zones, count.index)}" 29 | hooks = join("\n", data.template_file.master-hooks.*.rendered) 30 | } 31 | } 32 | 33 | data "template_file" "master-additional-sgs" { 34 | count = var.master-additional-sgs-count 35 | 36 | template = " - $${sg-id}" 37 | 38 | vars = { 39 | sg-id = element(var.master-additional-sgs, count.index) 40 | } 41 | } 42 | 43 | data "template_file" "master-cloud-labels" { 44 | count = length(keys(var.master-cloud-labels)) 45 | 46 | template = < 0 ? "additionalSecurityGroups:" : ""} 90 | ${join("\n", data.template_file.bastion-additional-sgs.*.rendered)} 91 | EOF 92 | 93 | image = var.bastion-image 94 | type = var.bastion-machine-type 95 | max-size = var.max-bastions 96 | min-size = var.min-bastions 97 | role = "Bastion" 98 | volume-size = var.bastion-volume-size 99 | volume-provisioned-iops = var.bastion-volume-provisioned-iops == "" ? "" : var.bastion-volume-provisioned-iops 100 | volume-type = var.bastion-volume-type 101 | ebs-optimized = var.bastion-ebs-optimized 102 | max-price = "" 103 | taints = "" 104 | subnets = join("\n", data.template_file.minion-subnets.*.rendered) 105 | hooks = join("\n", data.template_file.bastion-hooks.*.rendered) 106 | } 107 | } 108 | 109 | data "template_file" "bastion-additional-sgs" { 110 | count = var.bastion-additional-sgs-count 111 | 112 | template = " - $${sg-id}" 113 | 114 | vars = { 115 | sg-id = element(var.bastion-additional-sgs, count.index) 116 | } 117 | } 118 | 119 | data "template_file" "bastion-cloud-labels" { 120 | count = length(keys(var.bastion-cloud-labels)) 121 | 122 | template = < 0 ? "additionalSecurityGroups:" : ""} 165 | ${join("\n", data.template_file.minion-additional-sgs.*.rendered)} 166 | EOF 167 | 168 | image = var.minion-image 169 | type = var.minion-machine-type 170 | max-size = var.max-minions 171 | min-size = var.min-minions 172 | role = "Node" 173 | volume-size = var.minion-volume-size 174 | volume-provisioned-iops = var.minion-volume-provisioned-iops == "" ? "" : var.minion-volume-provisioned-iops 175 | volume-type = var.minion-volume-type 176 | ebs-optimized = var.minion-ebs-optimized 177 | max-price = "" 178 | taints = join("\n", data.template_file.minion-taints.*.rendered) 179 | subnets = join("\n", data.template_file.minion-subnets.*.rendered) 180 | hooks = join("\n", data.template_file.minion-hooks.*.rendered) 181 | } 182 | } 183 | 184 | data "template_file" "minion-taints" { 185 | count = length(var.minion-taints) 186 | 187 | template = " - ${element(var.minion-taints, count.index)}" 188 | } 189 | 190 | data "template_file" "minion-subnets" { 191 | count = length(var.availability-zones) 192 | template = " - $${az}" 193 | 194 | vars = { 195 | az = element(var.availability-zones, count.index) 196 | } 197 | } 198 | 199 | data "template_file" "minion-additional-sgs" { 200 | count = var.minion-additional-sgs-count 201 | 202 | template = " - $${sg-id}" 203 | 204 | vars = { 205 | sg-id = element(var.minion-additional-sgs, count.index) 206 | } 207 | } 208 | 209 | data "template_file" "minion-cloud-labels" { 210 | count = length(keys(var.minion-cloud-labels)) 211 | 212 | template = < ${path.module}/${var.cluster-name}-cluster-spec.yml 30 | ${aws_s3_bucket_object.cluster-spec.content} 31 | EOF 32 | FILEDUMP 33 | } 34 | 35 | // Let's wait for our newly created DNS zone to propagate 36 | provisioner "local-exec" { 37 | command = < ${path.module}/${var.cluster-name}-cluster-spec.yml 88 | ${aws_s3_bucket_object.cluster-spec.content} 89 | EOF 90 | FILEDUMP 91 | } 92 | 93 | provisioner "local-exec" { 94 | command = < 0 ? "additionalSecurityGroups:" : ""} 13 | ${join("\n", data.template_file.additional-sgs.*.rendered)} 14 | EOF 15 | 16 | image = var.image 17 | type = var.type 18 | max-size = var.max-size 19 | min-size = var.min-size 20 | role = "Node" 21 | volume-size = var.volume-size 22 | volume-provisioned-iops = var.volume-provisioned-iops == "" ? "" : var.volume-provisioned-iops 23 | volume-type = var.volume-type 24 | ebs-optimized = var.ebs-optimized 25 | max-price = "maxPrice: '${var.max-price}'" 26 | taints = join("\n", data.template_file.taints.*.rendered) 27 | subnets = join("\n", data.template_file.subnets.*.rendered) 28 | hooks = join("\n", data.template_file.hooks.*.rendered) 29 | } 30 | } 31 | 32 | data "template_file" "additional-sgs" { 33 | count = var.additional-sgs-count 34 | 35 | template = " - $${sg-id}" 36 | 37 | vars = { 38 | sg-id = element(var.additional-sgs, count.index) 39 | } 40 | } 41 | 42 | data "template_file" "cloud-labels" { 43 | count = length(keys(var.cloud-labels)) 44 | 45 | template = < ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 24 | ${aws_s3_bucket_object.ig-spec.content} 25 | EOF 26 | FILEDUMP 27 | } 28 | 29 | // Let's register our Kops cluster into remote state 30 | provisioner "local-exec" { 31 | command = < ${path.root}/.kops-ig-lock 38 | 39 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} create -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 40 | 41 | rmdir ${path.root}/.kops-ig-lock 42 | EOF 43 | } 44 | 45 | // Let's remove the ig spec from disk 46 | provisioner "local-exec" { 47 | command = "rm -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml" 48 | } 49 | 50 | depends_on = [aws_s3_bucket_object.ig-spec] 51 | } 52 | 53 | resource "null_resource" "ig-update" { 54 | triggers = { 55 | cluster_spec = data.template_file.ig-spec.rendered 56 | } 57 | 58 | provisioner "local-exec" { 59 | command = < ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 61 | ${data.template_file.ig-spec.rendered} 62 | EOF 63 | FILEDUMP 64 | } 65 | 66 | provisioner "local-exec" { 67 | command = < ${path.root}/.kops-ig-lock 74 | 75 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} \ 76 | replace -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 77 | 78 | rm -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 79 | 80 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} \ 81 | update cluster ${var.cluster-name} --yes 82 | 83 | KOPS_FEATURE_FLAGS="+DrainAndValidateRollingUpdate" \ 84 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} \ 85 | rolling-update cluster ${var.cluster-name} \ 86 | --node-interval="${var.update-interval}m" ${var.automatic-rollout == "true" ? "--yes" : ""}\ 87 | --instance-group="${var.name}" 88 | 89 | rmdir ${path.root}/.kops-ig-lock 90 | EOF 91 | } 92 | 93 | depends_on = [null_resource.ig] 94 | } 95 | -------------------------------------------------------------------------------- /aws/ig/outputs.tf: -------------------------------------------------------------------------------- 1 | output "created" { 2 | value = null_resource.ig.id 3 | } 4 | 5 | // ID of the AWS autoscaling group corresponding to this kops instance group 6 | output "asg-name" { 7 | value = element(data.aws_autoscaling_groups.ig.names, 0) 8 | } 9 | -------------------------------------------------------------------------------- /aws/ig/templates/spec.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kops/v1alpha2 2 | kind: InstanceGroup 3 | metadata: 4 | labels: 5 | kops.k8s.io/cluster: ${cluster-name} 6 | name: ${name} 7 | spec: 8 | cloudLabels: 9 | ${cloud-labels} 10 | nodeLabels: 11 | ${node-labels} 12 | associatePublicIp: ${public} 13 | ${additional-sgs} 14 | image: ${image} 15 | machineType: ${type} 16 | maxSize: ${max-size} 17 | minSize: ${min-size} 18 | role: ${role} 19 | rootVolumeSize: ${volume-size} 20 | rootVolumeType: ${volume-type} 21 | rootProvisionedIops: ${volume-provisioned-iops} 22 | rootVolumeOptimization: ${ebs-optimized} 23 | ${max-price} 24 | taints: 25 | ${taints} 26 | subnets: 27 | ${subnets} 28 | hooks: 29 | ${hooks} 30 | -------------------------------------------------------------------------------- /aws/ig/variables.tf: -------------------------------------------------------------------------------- 1 | # Dependency hooks 2 | variable "master-up" { 3 | type = "string" 4 | description = "Terraform dependency hook to wait for the master cluster to be up before creating instance groups" 5 | } 6 | 7 | # Kops env. overrides 8 | variable "nodeup-url-env" { 9 | type = "string" 10 | description = "NODEUP_URL env. variable override for testing custom builds of nodeup" 11 | 12 | default = "" 13 | } 14 | 15 | variable "aws-profile" { 16 | type = "string" 17 | description = "Name of the AWS profile in ~/.aws/credentials or ~/.aws/config to use" 18 | 19 | default = "default" 20 | } 21 | 22 | # Instance group parameters 23 | variable "cluster-name" { 24 | type = "string" 25 | description = "The name of the Kops cluster the instance group belongs to" 26 | } 27 | 28 | variable "name" { 29 | type = "string" 30 | description = "The name of the instance group" 31 | } 32 | 33 | variable "kops-state-bucket" { 34 | type = "string" 35 | description = "" 36 | } 37 | 38 | variable "automatic-rollout" { 39 | type = "string" 40 | description = "If set to true, a rolling update of the instance group will be triggered when its spec is modified" 41 | 42 | default = "false" 43 | } 44 | 45 | variable "update-interval" { 46 | type = "string" 47 | description = "Rolling update interval" 48 | 49 | default = 8 50 | } 51 | 52 | # Networking & Security 53 | variable "visibility" { 54 | type = "string" 55 | description = "Visibility (public|private) of the instance group (default: private)" 56 | 57 | default = "private" 58 | } 59 | 60 | variable "subnets" { 61 | type = "list" 62 | description = "Subnets this instance group should span" 63 | } 64 | 65 | variable "additional-sgs" { 66 | type = "list" 67 | description = "A list of additional security groups to add to this instance" 68 | 69 | default = [] 70 | } 71 | 72 | variable "additional-sgs-count" { 73 | type = "string" 74 | description = "Number of additional security groups to add" 75 | 76 | default = 0 77 | } 78 | 79 | # Node config 80 | variable "image" { 81 | type = "string" 82 | description = "AMI id to use for the nodes" 83 | } 84 | 85 | variable "type" { 86 | type = "string" 87 | description = "EC2 instance type to run our nodes onto" 88 | } 89 | 90 | variable "volume-size" { 91 | type = "string" 92 | description = "Size of our nodes' root volume, in GB (default: 10)" 93 | 94 | default = "10" 95 | } 96 | 97 | variable "volume-provisioned-iops" { 98 | type = "string" 99 | description = "Nodes volume provisioned IOPS, if applicable" 100 | 101 | default = "" 102 | } 103 | 104 | variable "volume-type" { 105 | type = "string" 106 | description = "Nodes volume type (io1/gp2), defaults to gp2" 107 | 108 | default = "gp2" 109 | } 110 | 111 | variable "ebs-optimized" { 112 | type = "string" 113 | description = "Boolean (true or false) indicating whether our nodes should be EBS optimized" 114 | 115 | default = "false" 116 | } 117 | 118 | variable "max-price" { 119 | type = "string" 120 | description = "If set, this group will use spot instances with the specified max-price" 121 | 122 | default = "" 123 | } 124 | 125 | variable "hooks" { 126 | type = "list" 127 | description = "Docker/Systemd hooks to add to this instance group (add 2 spaces at the beginning of each line for indentation. Also, you'll need the '-' (dash) to indicate that this hook is part of a list.)" 128 | 129 | default = [] 130 | } 131 | 132 | # ASG configuration 133 | variable "max-size" { 134 | type = "string" 135 | description = "Group max size" 136 | 137 | default = "5" 138 | } 139 | 140 | variable "min-size" { 141 | type = "string" 142 | description = "Group min size" 143 | 144 | default = "1" 145 | } 146 | 147 | # Labels 148 | variable "taints" { 149 | type = "list" 150 | description = "List of taints to add to the nodes" 151 | 152 | default = [] 153 | } 154 | 155 | variable "cloud-labels" { 156 | type = "map" 157 | description = "(Flat) map of cloud labels" 158 | 159 | default = {} 160 | } 161 | 162 | variable "node-labels" { 163 | type = "map" 164 | description = "(Flat) map of node labels" 165 | 166 | default = {} 167 | } 168 | -------------------------------------------------------------------------------- /aws/test/data.tf: -------------------------------------------------------------------------------- 1 | # AWS account information 2 | data "aws_caller_identity" "current" {} 3 | 4 | data "aws_region" "main" { 5 | current = true 6 | } 7 | 8 | # CoreOS base AMI 9 | variable "coreos-ami-owner-id" { 10 | description = "The ID of the owner of the CoreOS image you want to use on the AWS marketplace (or yours if you're using your own AMI)." 11 | 12 | # CoreOS' official AWS id 13 | default = "595879546273" 14 | type = "string" 15 | } 16 | 17 | variable "coreos-ami-pattern" { 18 | description = "The AMI pattern to use (it can be a full name or contain wildcards, default to the last release of CoreOS on the stable channel)." 19 | 20 | # Useful to change that to also run tests against the beta and alpha version of Container Linux 21 | default = "CoreOS-stable-*" 22 | type = "string" 23 | } 24 | 25 | data "aws_ami" "coreos-stable" { 26 | most_recent = true 27 | 28 | filter { 29 | name = "name" 30 | values = ["${var.coreos-ami-pattern}"] 31 | } 32 | 33 | filter { 34 | name = "virtualization-type" 35 | values = ["hvm"] 36 | } 37 | 38 | owners = ["${var.coreos-ami-owner-id}"] 39 | } 40 | 41 | data "aws_route53_zone" "test" { 42 | name = "${var.domain}" 43 | private_zone = "false" 44 | } 45 | -------------------------------------------------------------------------------- /aws/test/dns.tf: -------------------------------------------------------------------------------- 1 | resource "aws_route53_record" "pong" { 2 | zone_id = "${data.aws_route53_zone.test.id}" 3 | name = "pong" 4 | type = "A" 5 | 6 | alias { 7 | name = "${module.test-cluster.ingress-elb-dns-name}" 8 | zone_id = "${module.test-cluster.ingress-elb-zone-id}" 9 | evaluate_target_health = "false" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /aws/test/iam-kops.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * IAM policies for kops are defined here, according to the "kops-policies" variable 3 | */ 4 | 5 | # Kops 6 | resource "aws_iam_group" "kops" { 7 | name = "kops-test" 8 | path = "/" 9 | } 10 | 11 | resource "aws_iam_group_policy_attachment" "kops" { 12 | count = "${length(var.kops-policies)}" 13 | group = "${aws_iam_group.kops.name}" 14 | policy_arn = "arn:aws:iam::aws:policy/${element(var.kops-policies, count.index)}" 15 | } 16 | 17 | resource "aws_iam_group_membership" "kops" { 18 | name = "kops-membership" 19 | group = "${aws_iam_group.kops.name}" 20 | users = ["${aws_iam_user.kops.name}"] 21 | } 22 | 23 | resource "aws_iam_user" "kops" { 24 | name = "kops-test" 25 | path = "/" 26 | } 27 | -------------------------------------------------------------------------------- /aws/test/k8s/backend/pong.yaml: -------------------------------------------------------------------------------- 1 | # Just an example service to confirm that requests are routed to pods 2 | # accordingly 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: pong 7 | labels: 8 | app: pong 9 | spec: 10 | type: ClusterIP 11 | selector: 12 | app: pong 13 | ports: 14 | - protocol: TCP 15 | port: 80 16 | targetPort: 80 17 | name: http 18 | --- 19 | apiVersion: extensions/v1beta1 20 | kind: Ingress 21 | metadata: 22 | name: pong 23 | annotations: 24 | kubernetes.io/tls-acme: "true" 25 | spec: 26 | tls: 27 | - secretName: tls-pong 28 | hosts: 29 | - pong.${domain} 30 | rules: 31 | - host: pong.${domain} 32 | http: 33 | paths: 34 | - path: / 35 | backend: 36 | serviceName: pong 37 | servicePort: http 38 | --- 39 | apiVersion: policy/v1beta1 40 | kind: PodDisruptionBudget 41 | metadata: 42 | name: pong 43 | spec: 44 | minAvailable: 1 45 | selector: 46 | matchLabels: 47 | app: pong 48 | --- 49 | apiVersion: autoscaling/v1 50 | kind: HorizontalPodAutoscaler 51 | metadata: 52 | name: pong 53 | spec: 54 | scaleTargetRef: 55 | apiVersion: extensions/v1beta1 56 | kind: Deployment 57 | name: pong 58 | minReplicas: 2 59 | maxReplicas: 10 60 | targetCPUUtilizationPercentage: 30 61 | --- 62 | kind: Deployment 63 | apiVersion: extensions/v1beta1 64 | metadata: 65 | name: pong 66 | labels: 67 | app: pong 68 | spec: 69 | revisionHistoryLimit: 1 70 | selector: 71 | matchLabels: 72 | app: pong 73 | template: 74 | metadata: 75 | labels: 76 | app: pong 77 | spec: 78 | terminationGracePeriodSeconds: 10 79 | nodeSelector: 80 | duty: webserver 81 | containers: 82 | - image: nginx:alpine 83 | name: nginx 84 | imagePullPolicy: Always 85 | resources: 86 | limits: 87 | memory: 200Mi 88 | requests: 89 | cpu: 300m 90 | memory: 200Mi 91 | livenessProbe: 92 | httpGet: 93 | scheme: HTTP 94 | port: 80 95 | readinessProbe: 96 | httpGet: 97 | scheme: HTTP 98 | port: 80 99 | ports: 100 | - name: http 101 | containerPort: 80 102 | protocol: TCP 103 | affinity: 104 | podAntiAffinity: 105 | preferredDuringSchedulingIgnoredDuringExecution: 106 | - weight: 100 107 | podAffinityTerm: 108 | labelSelector: 109 | matchExpressions: 110 | - key: app 111 | operator: In 112 | values: ["pong"] 113 | topologyKey: kubernetes.io/hostname 114 | - weight: 50 115 | podAffinityTerm: 116 | labelSelector: 117 | matchExpressions: 118 | - key: app 119 | operator: In 120 | values: ["pong"] 121 | topologyKey: failure-domain.beta.kubernetes.io/zone 122 | -------------------------------------------------------------------------------- /aws/test/k8s/ingress/default-backend.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: default-backend 5 | namespace: ingress 6 | labels: 7 | k8s-app: default-http-backend 8 | k8s-addon: ingress-nginx.addons.k8s.io 9 | app: default-backend 10 | spec: 11 | ports: 12 | - port: 80 13 | targetPort: http 14 | selector: 15 | app: default-backend 16 | --- 17 | apiVersion: policy/v1beta1 18 | kind: PodDisruptionBudget 19 | metadata: 20 | name: default-backend 21 | namespace: ingress 22 | spec: 23 | minAvailable: 1 24 | selector: 25 | matchLabels: 26 | app: default-backend 27 | --- 28 | apiVersion: autoscaling/v1 29 | kind: HorizontalPodAutoscaler 30 | metadata: 31 | name: default-backend 32 | namespace: ingress 33 | spec: 34 | scaleTargetRef: 35 | apiVersion: extensions/v1beta1 36 | kind: Deployment 37 | name: default-backend 38 | minReplicas: 2 39 | maxReplicas: 5 40 | targetCPUUtilizationPercentage: 70 41 | --- 42 | kind: Deployment 43 | apiVersion: extensions/v1beta1 44 | metadata: 45 | name: default-backend 46 | namespace: ingress 47 | labels: 48 | k8s-app: default-http-backend 49 | k8s-addon: ingress-nginx.addons.k8s.io 50 | app: default-backend 51 | spec: 52 | revisionHistoryLimit: 10 53 | template: 54 | metadata: 55 | labels: 56 | app: default-backend 57 | spec: 58 | nodeSelector: 59 | duty: intake 60 | terminationGracePeriodSeconds: 60 61 | containers: 62 | - name: default-http-backend 63 | image: gcr.io/google_containers/defaultbackend:1.4 64 | livenessProbe: 65 | httpGet: 66 | path: /healthz 67 | port: 8080 68 | scheme: HTTP 69 | initialDelaySeconds: 30 70 | timeoutSeconds: 5 71 | resources: 72 | limits: 73 | cpu: 10m 74 | memory: 20Mi 75 | requests: 76 | cpu: 10m 77 | memory: 20Mi 78 | ports: 79 | - name: http 80 | containerPort: 8080 81 | protocol: TCP 82 | restartPolicy: Always 83 | affinity: 84 | podAntiAffinity: 85 | preferredDuringSchedulingIgnoredDuringExecution: 86 | - weight: 100 87 | podAffinityTerm: 88 | labelSelector: 89 | matchExpressions: 90 | - key: app 91 | operator: In 92 | values: ["default-backend"] 93 | topologyKey: kubernetes.io/hostname 94 | - weight: 50 95 | podAffinityTerm: 96 | labelSelector: 97 | matchExpressions: 98 | - key: app 99 | operator: In 100 | values: ["default-backend"] 101 | topologyKey: failure-domain.beta.kubernetes.io/zone 102 | -------------------------------------------------------------------------------- /aws/test/k8s/ingress/kube-lego.yaml: -------------------------------------------------------------------------------- 1 | # Note: kube-lego fetches certificates from Let's Encrypt for you ingress 2 | # domains automatically... however, we wouldn't recommend using it for 3 | # production-critical workloads, unless you really don't want to pay for a 4 | # wildcard certificate... 5 | # --- 6 | 7 | # ServiceAccount and Roles 8 | apiVersion: v1 9 | kind: ServiceAccount 10 | metadata: 11 | name: kube-lego 12 | namespace: ingress 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1beta1 15 | kind: ClusterRole 16 | metadata: 17 | name: kube-lego 18 | rules: 19 | - apiGroups: 20 | - extensions 21 | resources: 22 | - ingresses 23 | verbs: 24 | - get 25 | - create 26 | - update 27 | - delete 28 | - patch 29 | - list 30 | - watch 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - secrets 35 | verbs: 36 | - get 37 | - create 38 | - update 39 | - apiGroups: 40 | - "" 41 | resources: 42 | - services 43 | verbs: 44 | - create 45 | - get 46 | - delete 47 | --- 48 | apiVersion: rbac.authorization.k8s.io/v1beta1 49 | kind: ClusterRoleBinding 50 | metadata: 51 | name: kube-lego 52 | roleRef: 53 | apiGroup: rbac.authorization.k8s.io 54 | kind: ClusterRole 55 | name: kube-lego 56 | subjects: 57 | - kind: ServiceAccount 58 | name: kube-lego 59 | namespace: ingress 60 | --- 61 | 62 | # The actual lego deployment 63 | apiVersion: extensions/v1beta1 64 | kind: Deployment 65 | metadata: 66 | name: kube-lego 67 | namespace: ingress 68 | spec: 69 | template: 70 | metadata: 71 | labels: 72 | app: kube-lego 73 | spec: 74 | serviceAccountName: kube-lego 75 | nodeSelector: 76 | duty: intake 77 | containers: 78 | - name: kube-lego 79 | image: jetstack/kube-lego:canary 80 | imagePullPolicy: Always 81 | ports: 82 | - containerPort: 8080 83 | env: 84 | - name: LEGO_EMAIL 85 | value: "karch-tests-01@mailinator.com" 86 | - name: LEGO_URL 87 | value: "https://acme-v01.api.letsencrypt.org/directory" 88 | - name: LEGO_NAMESPACE 89 | value: ingress 90 | - name: LEGO_POD_IP 91 | valueFrom: 92 | fieldRef: 93 | fieldPath: status.podIP 94 | readinessProbe: 95 | httpGet: 96 | path: /healthz 97 | port: 8080 98 | initialDelaySeconds: 5 99 | timeoutSeconds: 1 100 | -------------------------------------------------------------------------------- /aws/test/k8s/ingress/nginx-ingress-controller-config.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: nginx-conf-template 5 | namespace: ingress 6 | data: 7 | nginx.tmpl: | 8 | {{ $all := . }} 9 | {{ $servers := .Servers }} 10 | {{ $cfg := .Cfg }} 11 | {{ $IsIPV6Enabled := .IsIPV6Enabled }} 12 | {{ $healthzURI := .HealthzURI }} 13 | {{ $backends := .Backends }} 14 | {{ $proxyHeaders := .ProxySetHeaders }} 15 | {{ $addHeaders := .AddHeaders }} 16 | 17 | {{ if $cfg.EnableModsecurity }} 18 | load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; 19 | {{ end }} 20 | 21 | {{ if $cfg.EnableOpentracing }} 22 | load_module /etc/nginx/modules/ngx_http_opentracing_module.so; 23 | {{ end }} 24 | 25 | {{ if (and $cfg.EnableOpentracing (ne $cfg.ZipkinCollectorHost "")) }} 26 | load_module /etc/nginx/modules/ngx_http_zipkin_module.so; 27 | {{ end }} 28 | 29 | daemon off; 30 | 31 | worker_processes {{ $cfg.WorkerProcesses }}; 32 | pid /run/nginx.pid; 33 | {{ if ne .MaxOpenFiles 0 }} 34 | worker_rlimit_nofile {{ .MaxOpenFiles }}; 35 | {{ end}} 36 | 37 | {{/* http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout */}} 38 | {{/* avoid waiting too long during a reload */}} 39 | worker_shutdown_timeout {{ $cfg.WorkerShutdownTimeout }} ; 40 | 41 | events { 42 | multi_accept off; 43 | accept_mutex off; 44 | worker_connections {{ $cfg.MaxWorkerConnections }}; 45 | } 46 | 47 | http { 48 | {{/* we use the value of the header X-Forwarded-For to be able to use the geo_ip module */}} 49 | {{ if $cfg.UseProxyProtocol }} 50 | real_ip_header proxy_protocol; 51 | {{ else }} 52 | real_ip_header {{ $cfg.ForwardedForHeader }}; 53 | {{ end }} 54 | 55 | real_ip_recursive on; 56 | {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }} 57 | set_real_ip_from {{ $trusted_ip }}; 58 | {{ end }} 59 | 60 | {{/* databases used to determine the country depending on the client IP address */}} 61 | {{/* http://nginx.org/en/docs/http/ngx_http_geoip_module.html */}} 62 | {{/* this is require to calculate traffic for individual country using GeoIP in the status page */}} 63 | geoip_country /etc/nginx/GeoIP.dat; 64 | geoip_city /etc/nginx/GeoLiteCity.dat; 65 | geoip_proxy_recursive on; 66 | 67 | {{ if $cfg.EnableVtsStatus }} 68 | vhost_traffic_status_zone shared:vhost_traffic_status:{{ $cfg.VtsStatusZoneSize }}; 69 | vhost_traffic_status_filter_by_set_key {{ $cfg.VtsDefaultFilterKey }}; 70 | {{ end }} 71 | 72 | sendfile on; 73 | 74 | aio threads; 75 | aio_write on; 76 | 77 | tcp_nopush on; 78 | tcp_nodelay on; 79 | 80 | log_subrequest on; 81 | 82 | reset_timedout_connection on; 83 | 84 | keepalive_timeout {{ $cfg.KeepAlive }}s; 85 | keepalive_requests {{ $cfg.KeepAliveRequests }}; 86 | 87 | client_header_buffer_size {{ $cfg.ClientHeaderBufferSize }}; 88 | client_header_timeout {{ $cfg.ClientHeaderTimeout }}s; 89 | large_client_header_buffers {{ $cfg.LargeClientHeaderBuffers }}; 90 | client_body_buffer_size {{ $cfg.ClientBodyBufferSize }}; 91 | client_body_timeout {{ $cfg.ClientBodyTimeout }}s; 92 | 93 | http2_max_field_size {{ $cfg.HTTP2MaxFieldSize }}; 94 | http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; 95 | 96 | types_hash_max_size 2048; 97 | server_names_hash_max_size {{ $cfg.ServerNameHashMaxSize }}; 98 | server_names_hash_bucket_size {{ $cfg.ServerNameHashBucketSize }}; 99 | map_hash_bucket_size {{ $cfg.MapHashBucketSize }}; 100 | 101 | proxy_headers_hash_max_size {{ $cfg.ProxyHeadersHashMaxSize }}; 102 | proxy_headers_hash_bucket_size {{ $cfg.ProxyHeadersHashBucketSize }}; 103 | 104 | variables_hash_bucket_size {{ $cfg.VariablesHashBucketSize }}; 105 | variables_hash_max_size {{ $cfg.VariablesHashMaxSize }}; 106 | 107 | underscores_in_headers {{ if $cfg.EnableUnderscoresInHeaders }}on{{ else }}off{{ end }}; 108 | ignore_invalid_headers {{ if $cfg.IgnoreInvalidHeaders }}on{{ else }}off{{ end }}; 109 | 110 | {{ if $cfg.EnableOpentracing }} 111 | opentracing on; 112 | {{ end }} 113 | 114 | {{ if (and $cfg.EnableOpentracing (ne $cfg.ZipkinCollectorHost "")) }} 115 | zipkin_collector_host {{ $cfg.ZipkinCollectorHost }}; 116 | zipkin_collector_port {{ $cfg.ZipkinCollectorPort }}; 117 | zipkin_service_name {{ $cfg.ZipkinServiceName }}; 118 | {{ end }} 119 | 120 | include /etc/nginx/mime.types; 121 | default_type text/html; 122 | 123 | {{ if $cfg.EnableBrotli }} 124 | brotli on; 125 | brotli_comp_level {{ $cfg.BrotliLevel }}; 126 | brotli_types {{ $cfg.BrotliTypes }}; 127 | {{ end }} 128 | 129 | {{ if $cfg.UseGzip }} 130 | gzip on; 131 | gzip_comp_level 5; 132 | gzip_http_version 1.1; 133 | gzip_min_length 256; 134 | gzip_types {{ $cfg.GzipTypes }}; 135 | gzip_proxied any; 136 | gzip_vary on; 137 | {{ end }} 138 | 139 | # Custom headers for response 140 | {{ range $k, $v := $addHeaders }} 141 | add_header {{ $k }} "{{ $v }}"; 142 | {{ end }} 143 | 144 | server_tokens {{ if $cfg.ShowServerTokens }}on{{ else }}off{{ end }}; 145 | 146 | # disable warnings 147 | uninitialized_variable_warn off; 148 | 149 | # Additional available variables: 150 | # $namespace 151 | # $ingress_name 152 | # $service_name 153 | log_format upstreaminfo {{ if $cfg.LogFormatEscapeJSON }}escape=json {{ end }}'{{ buildLogFormatUpstream $cfg }}'; 154 | 155 | {{/* map urls that should not appear in access.log */}} 156 | {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} 157 | map $request_uri $loggable { 158 | {{ range $reqUri := $cfg.SkipAccessLogURLs }} 159 | {{ $reqUri }} 0;{{ end }} 160 | default 1; 161 | } 162 | 163 | {{ if $cfg.DisableAccessLog }} 164 | access_log off; 165 | {{ else }} 166 | access_log {{ $cfg.AccessLogPath }} upstreaminfo if=$loggable; 167 | {{ end }} 168 | error_log {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }}; 169 | 170 | {{ buildResolvers $cfg.Resolver }} 171 | 172 | {{/* Whenever nginx proxies a request without a "Connection" header, the "Connection" header is set to "close" */}} 173 | {{/* when making the target request. This means that you cannot simply use */}} 174 | {{/* "proxy_set_header Connection $http_connection" for WebSocket support because in this case, the */}} 175 | {{/* "Connection" header would be set to "" whenever the original request did not have a "Connection" header, */}} 176 | {{/* which would mean no "Connection" header would be in the target request. Since this would deviate from */}} 177 | {{/* normal nginx behavior we have to use this approach. */}} 178 | # Retain the default nginx handling of requests without a "Connection" header 179 | map $http_upgrade $connection_upgrade { 180 | default upgrade; 181 | '' close; 182 | } 183 | 184 | map {{ buildForwardedFor $cfg.ForwardedForHeader }} $the_real_ip { 185 | {{ if $cfg.UseProxyProtocol }} 186 | # Get IP address from Proxy Protocol 187 | default $proxy_protocol_addr; 188 | {{ else }} 189 | default $remote_addr; 190 | {{ end }} 191 | } 192 | 193 | # trust http_x_forwarded_proto headers correctly indicate ssl offloading 194 | map $http_x_forwarded_proto $pass_access_scheme { 195 | default $http_x_forwarded_proto; 196 | '' $scheme; 197 | } 198 | 199 | map $http_x_forwarded_port $pass_server_port { 200 | default $http_x_forwarded_port; 201 | '' $server_port; 202 | } 203 | 204 | map $http_x_forwarded_host $best_http_host { 205 | default $http_x_forwarded_host; 206 | '' $this_host; 207 | } 208 | 209 | {{ if $all.IsSSLPassthroughEnabled }} 210 | # map port {{ $all.ListenPorts.SSLProxy }} to 443 for header X-Forwarded-Port 211 | map $pass_server_port $pass_port { 212 | {{ $all.ListenPorts.SSLProxy }} 443; 213 | default $pass_server_port; 214 | } 215 | {{ else }} 216 | map $pass_server_port $pass_port { 217 | 443 443; 218 | default $pass_server_port; 219 | } 220 | {{ end }} 221 | 222 | # Obtain best http host 223 | map $http_host $this_host { 224 | default $http_host; 225 | '' $host; 226 | } 227 | 228 | {{ if $cfg.ComputeFullForwardedFor }} 229 | # We can't use $proxy_add_x_forwarded_for because the realip module 230 | # replaces the remote_addr too soon 231 | map $http_x_forwarded_for $full_x_forwarded_for { 232 | {{ if $all.Cfg.UseProxyProtocol }} 233 | default "$http_x_forwarded_for, $proxy_protocol_addr"; 234 | '' "$proxy_protocol_addr"; 235 | {{ else }} 236 | default "$http_x_forwarded_for, $realip_remote_addr"; 237 | '' "$realip_remote_addr"; 238 | {{ end}} 239 | } 240 | {{ end }} 241 | 242 | server_name_in_redirect off; 243 | port_in_redirect off; 244 | 245 | ssl_protocols {{ $cfg.SSLProtocols }}; 246 | 247 | # turn on session caching to drastically improve performance 248 | {{ if $cfg.SSLSessionCache }} 249 | ssl_session_cache builtin:1000 shared:SSL:{{ $cfg.SSLSessionCacheSize }}; 250 | ssl_session_timeout {{ $cfg.SSLSessionTimeout }}; 251 | {{ end }} 252 | 253 | # allow configuring ssl session tickets 254 | ssl_session_tickets {{ if $cfg.SSLSessionTickets }}on{{ else }}off{{ end }}; 255 | 256 | {{ if not (empty $cfg.SSLSessionTicketKey ) }} 257 | ssl_session_ticket_key /etc/nginx/tickets.key; 258 | {{ end }} 259 | 260 | # slightly reduce the time-to-first-byte 261 | ssl_buffer_size {{ $cfg.SSLBufferSize }}; 262 | 263 | {{ if not (empty $cfg.SSLCiphers) }} 264 | # allow configuring custom ssl ciphers 265 | ssl_ciphers '{{ $cfg.SSLCiphers }}'; 266 | ssl_prefer_server_ciphers on; 267 | {{ end }} 268 | 269 | {{ if not (empty $cfg.SSLDHParam) }} 270 | # allow custom DH file http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam 271 | ssl_dhparam {{ $cfg.SSLDHParam }}; 272 | {{ end }} 273 | 274 | {{ if not $cfg.EnableDynamicTLSRecords }} 275 | ssl_dyn_rec_size_lo 0; 276 | {{ end }} 277 | 278 | ssl_ecdh_curve {{ $cfg.SSLECDHCurve }}; 279 | 280 | {{ if .CustomErrors }} 281 | # Custom error pages 282 | proxy_intercept_errors on; 283 | {{ end }} 284 | 285 | {{ range $errCode := $cfg.CustomHTTPErrors }} 286 | error_page {{ $errCode }} = @custom_{{ $errCode }};{{ end }} 287 | 288 | proxy_ssl_session_reuse on; 289 | 290 | {{ if $cfg.AllowBackendServerHeader }} 291 | proxy_pass_header Server; 292 | {{ end }} 293 | 294 | {{ if not (empty $cfg.HTTPSnippet) }} 295 | # Custom code snippet configured in the configuration configmap 296 | {{ $cfg.HTTPSnippet }} 297 | {{ end }} 298 | 299 | {{ range $name, $upstream := $backends }} 300 | {{ if eq $upstream.SessionAffinity.AffinityType "cookie" }} 301 | upstream sticky-{{ $upstream.Name }} { 302 | sticky hash={{ $upstream.SessionAffinity.CookieSessionAffinity.Hash }} name={{ $upstream.SessionAffinity.CookieSessionAffinity.Name }} httponly; 303 | 304 | {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} 305 | keepalive {{ $cfg.UpstreamKeepaliveConnections }}; 306 | {{ end }} 307 | 308 | {{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }}; 309 | {{ end }} 310 | 311 | } 312 | 313 | {{ end }} 314 | 315 | 316 | upstream {{ $upstream.Name }} { 317 | # Load balance algorithm; empty for round robin, which is the default 318 | {{ if ne $cfg.LoadBalanceAlgorithm "round_robin" }} 319 | {{ $cfg.LoadBalanceAlgorithm }}; 320 | {{ end }} 321 | 322 | {{ if $upstream.UpstreamHashBy }} 323 | hash {{ $upstream.UpstreamHashBy }} consistent; 324 | {{ end }} 325 | 326 | {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} 327 | keepalive {{ $cfg.UpstreamKeepaliveConnections }}; 328 | {{ end }} 329 | 330 | {{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }}; 331 | {{ end }} 332 | } 333 | 334 | {{ end }} 335 | 336 | {{/* build the maps that will be use to validate the Whitelist */}} 337 | {{ range $index, $server := $servers }} 338 | {{ range $location := $server.Locations }} 339 | {{ $path := buildLocation $location }} 340 | 341 | {{ if isLocationAllowed $location }} 342 | {{ if gt (len $location.Whitelist.CIDR) 0 }} 343 | 344 | # Deny for {{ print $server.Hostname $path }} 345 | geo $the_real_ip {{ buildDenyVariable (print $server.Hostname "_" $path) }} { 346 | default 1; 347 | 348 | {{ range $ip := $location.Whitelist.CIDR }} 349 | {{ $ip }} 0;{{ end }} 350 | } 351 | {{ end }} 352 | {{ end }} 353 | {{ end }} 354 | {{ end }} 355 | 356 | {{ range $rl := (filterRateLimits $servers ) }} 357 | # Ratelimit {{ $rl.Name }} 358 | geo $the_real_ip $whitelist_{{ $rl.ID }} { 359 | default 0; 360 | {{ range $ip := $rl.Whitelist }} 361 | {{ $ip }} 1;{{ end }} 362 | } 363 | 364 | # Ratelimit {{ $rl.Name }} 365 | map $whitelist_{{ $rl.ID }} $limit_{{ $rl.ID }} { 366 | 0 {{ $cfg.LimitConnZoneVariable }}; 367 | 1 ""; 368 | } 369 | {{ end }} 370 | 371 | {{/* build all the required rate limit zones. Each annotation requires a dedicated zone */}} 372 | {{/* 1MB -> 16 thousand 64-byte states or about 8 thousand 128-byte states */}} 373 | {{ range $zone := (buildRateLimitZones $servers) }} 374 | {{ $zone }} 375 | {{ end }} 376 | 377 | {{/* Build server redirects (from/to www) */}} 378 | {{ range $hostname, $to := .RedirectServers }} 379 | server { 380 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 381 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 382 | listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl; 383 | {{ else }} 384 | listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 385 | listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl; 386 | {{ end }} 387 | {{ if $IsIPV6Enabled }} 388 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 389 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 390 | listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}; 391 | {{ else }} 392 | listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 393 | listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}; 394 | {{ end }} 395 | {{ end }} 396 | server_name {{ $hostname }}; 397 | 398 | {{ if ne $all.ListenPorts.HTTPS 443 }} 399 | {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} 400 | return {{ $all.Cfg.HTTPRedirectCode }} $scheme://{{ $to }}{{ $redirect_port }}$request_uri; 401 | {{ else }} 402 | return {{ $all.Cfg.HTTPRedirectCode }} $scheme://{{ $to }}$request_uri; 403 | {{ end }} 404 | } 405 | {{ end }} 406 | 407 | {{ range $index, $server := $servers }} 408 | 409 | ## start server {{ $server.Hostname }} 410 | server { 411 | server_name {{ $server.Hostname }} {{ $server.Alias }}; 412 | {{ template "SERVER" serverConfig $all $server }} 413 | 414 | {{ if not (empty $cfg.ServerSnippet) }} 415 | # Custom code snippet configured in the configuration configmap 416 | {{ $cfg.ServerSnippet }} 417 | {{ end }} 418 | 419 | {{ template "CUSTOM_ERRORS" $all }} 420 | } 421 | ## end server {{ $server.Hostname }} 422 | 423 | {{ end }} 424 | 425 | # default server, used for NGINX healthcheck and access to nginx stats 426 | server { 427 | # Use the port {{ $all.ListenPorts.Status }} (random value just to avoid known ports) as default port for nginx. 428 | # Changing this value requires a change in: 429 | # https://github.com/kubernetes/ingress-nginx/blob/master/controllers/nginx/pkg/cmd/controller/nginx.go 430 | listen {{ $all.ListenPorts.Status }} default_server reuseport backlog={{ $all.BacklogSize }}; 431 | {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Status }} default_server reuseport backlog={{ $all.BacklogSize }};{{ end }} 432 | set $proxy_upstream_name "-"; 433 | 434 | location {{ $healthzURI }} { 435 | access_log off; 436 | return 200; 437 | } 438 | 439 | location /nginx_status { 440 | set $proxy_upstream_name "internal"; 441 | 442 | {{ if $cfg.EnableVtsStatus }} 443 | vhost_traffic_status_display; 444 | vhost_traffic_status_display_format html; 445 | {{ else }} 446 | access_log off; 447 | stub_status on; 448 | {{ end }} 449 | } 450 | 451 | location / { 452 | {{ if .CustomErrors }} 453 | proxy_set_header X-Code 404; 454 | {{ end }} 455 | set $proxy_upstream_name "upstream-default-backend"; 456 | proxy_pass http://upstream-default-backend; 457 | } 458 | 459 | {{ template "CUSTOM_ERRORS" $all }} 460 | } 461 | } 462 | 463 | stream { 464 | log_format log_stream {{ $cfg.LogFormatStream }}; 465 | 466 | {{ if $cfg.DisableAccessLog }} 467 | access_log off; 468 | {{ else }} 469 | access_log {{ $cfg.AccessLogPath }} log_stream; 470 | {{ end }} 471 | 472 | error_log {{ $cfg.ErrorLogPath }}; 473 | 474 | # TCP services 475 | {{ range $i, $tcpServer := .TCPBackends }} 476 | upstream tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }} { 477 | {{ range $j, $endpoint := $tcpServer.Endpoints }} 478 | server {{ $endpoint.Address }}:{{ $endpoint.Port }}; 479 | {{ end }} 480 | } 481 | server { 482 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 483 | listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 484 | {{ else }} 485 | listen {{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 486 | {{ end }} 487 | {{ if $IsIPV6Enabled }} 488 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 489 | listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 490 | {{ else }} 491 | listen [::]:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 492 | {{ end }} 493 | {{ end }} 494 | proxy_timeout {{ $cfg.ProxyStreamTimeout }}; 495 | proxy_pass tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}; 496 | {{ if $tcpServer.Backend.ProxyProtocol.Encode }} 497 | proxy_protocol on; 498 | {{ end }} 499 | } 500 | 501 | {{ end }} 502 | 503 | # UDP services 504 | {{ range $i, $udpServer := .UDPBackends }} 505 | upstream udp-{{ $udpServer.Port }}-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }} { 506 | {{ range $j, $endpoint := $udpServer.Endpoints }} 507 | server {{ $endpoint.Address }}:{{ $endpoint.Port }}; 508 | {{ end }} 509 | } 510 | 511 | server { 512 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 513 | listen {{ $address }}:{{ $udpServer.Port }} udp; 514 | {{ else }} 515 | listen {{ $udpServer.Port }} udp; 516 | {{ end }} 517 | {{ if $IsIPV6Enabled }} 518 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 519 | listen {{ $address }}:{{ $udpServer.Port }} udp; 520 | {{ else }} 521 | listen [::]:{{ $udpServer.Port }} udp; 522 | {{ end }} 523 | {{ end }} 524 | proxy_responses {{ $cfg.ProxyStreamResponses }}; 525 | proxy_timeout {{ $cfg.ProxyStreamTimeout }}; 526 | proxy_pass udp-{{ $udpServer.Port }}-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }}; 527 | } 528 | 529 | {{ end }} 530 | } 531 | 532 | {{/* definition of templates to avoid repetitions */}} 533 | {{ define "CUSTOM_ERRORS" }} 534 | {{ $proxySetHeaders := .ProxySetHeaders }} 535 | {{ range $errCode := .Cfg.CustomHTTPErrors }} 536 | location @custom_{{ $errCode }} { 537 | internal; 538 | 539 | proxy_intercept_errors off; 540 | 541 | proxy_set_header X-Code {{ $errCode }}; 542 | proxy_set_header X-Format $http_accept; 543 | proxy_set_header X-Original-URI $request_uri; 544 | proxy_set_header X-Namespace $namespace; 545 | proxy_set_header X-Ingress-Name $ingress_name; 546 | proxy_set_header X-Service-Name $service_name; 547 | 548 | rewrite (.*) / break; 549 | proxy_pass http://upstream-default-backend; 550 | } 551 | {{ end }} 552 | {{ end }} 553 | 554 | {{/* CORS support from https://michielkalkman.com/snippets/nginx-cors-open-configuration.html */}} 555 | {{ define "CORS" }} 556 | {{ $cors := .CorsConfig }} 557 | # Cors Preflight methods needs additional options and different Return Code 558 | if ($request_method = 'OPTIONS') { 559 | add_header 'Access-Control-Allow-Origin' '{{ $cors.CorsAllowOrigin }}' always; 560 | {{ if $cors.CorsAllowCredentials }} add_header 'Access-Control-Allow-Credentials' '{{ $cors.CorsAllowCredentials }}' always; {{ end }} 561 | add_header 'Access-Control-Allow-Methods' '{{ $cors.CorsAllowMethods }}' always; 562 | add_header 'Access-Control-Allow-Headers' '{{ $cors.CorsAllowHeaders }}' always; 563 | add_header 'Access-Control-Max-Age' 1728000; 564 | add_header 'Content-Type' 'text/plain charset=UTF-8'; 565 | add_header 'Content-Length' 0; 566 | return 204; 567 | } 568 | 569 | add_header 'Access-Control-Allow-Origin' '{{ $cors.CorsAllowOrigin }}' always; 570 | {{ if $cors.CorsAllowCredentials }} add_header 'Access-Control-Allow-Credentials' '{{ $cors.CorsAllowCredentials }}' always; {{ end }} 571 | add_header 'Access-Control-Allow-Methods' '{{ $cors.CorsAllowMethods }}' always; 572 | add_header 'Access-Control-Allow-Headers' '{{ $cors.CorsAllowHeaders }}' always; 573 | 574 | {{ end }} 575 | 576 | {{/* definition of server-template to avoid repetitions with server-alias */}} 577 | {{ define "SERVER" }} 578 | {{ $all := .First }} 579 | {{ $server := .Second }} 580 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 581 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}}; 582 | {{ else }} 583 | listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}}; 584 | {{ end }} 585 | {{ if $all.IsIPV6Enabled }} 586 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 587 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; 588 | {{ else }} 589 | listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; 590 | {{ end }} 591 | {{ end }} 592 | set $proxy_upstream_name "-"; 593 | 594 | {{/* Listen on {{ $all.ListenPorts.SSLProxy }} because port {{ $all.ListenPorts.HTTPS }} is used in the TLS sni server */}} 595 | {{/* This listener must always have proxy_protocol enabled, because the SNI listener forwards on source IP info in it. */}} 596 | {{ if not (empty $server.SSLCertificate) }} 597 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 598 | listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 599 | {{ else }} 600 | listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 601 | {{ end }} 602 | {{ if $all.IsIPV6Enabled }} 603 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 604 | {{ if not (empty $server.SSLCertificate) }}listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 605 | {{ else }} 606 | {{ if not (empty $server.SSLCertificate) }}listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 607 | {{ end }} 608 | {{ end }} 609 | {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} 610 | # PEM sha: {{ $server.SSLPemChecksum }} 611 | ssl_certificate {{ $server.SSLCertificate }}; 612 | ssl_certificate_key {{ $server.SSLCertificate }}; 613 | {{ if not (empty $server.SSLFullChainCertificate)}} 614 | ssl_trusted_certificate {{ $server.SSLFullChainCertificate }}; 615 | ssl_stapling on; 616 | ssl_stapling_verify on; 617 | {{ end }} 618 | {{ end }} 619 | 620 | {{ if (and (not (empty $server.SSLCertificate)) $all.Cfg.HSTS) }} 621 | more_set_headers "Strict-Transport-Security: max-age={{ $all.Cfg.HSTSMaxAge }}{{ if $all.Cfg.HSTSIncludeSubdomains }}; includeSubDomains{{ end }};{{ if $all.Cfg.HSTSPreload }} preload{{ end }}"; 622 | {{ end }} 623 | 624 | 625 | {{ if not (empty $server.CertificateAuth.CAFileName) }} 626 | # PEM sha: {{ $server.CertificateAuth.PemSHA }} 627 | ssl_client_certificate {{ $server.CertificateAuth.CAFileName }}; 628 | ssl_verify_client {{ $server.CertificateAuth.VerifyClient }}; 629 | ssl_verify_depth {{ $server.CertificateAuth.ValidationDepth }}; 630 | {{ if not (empty $server.CertificateAuth.ErrorPage)}} 631 | error_page 495 496 = {{ $server.CertificateAuth.ErrorPage }}; 632 | {{ end }} 633 | {{ end }} 634 | 635 | {{ if not (empty $server.ServerSnippet) }} 636 | {{ $server.ServerSnippet }} 637 | {{ end }} 638 | 639 | {{ range $location := $server.Locations }} 640 | {{ $path := buildLocation $location }} 641 | {{ $authPath := buildAuthLocation $location }} 642 | 643 | {{ if not (empty $location.Rewrite.AppRoot)}} 644 | if ($uri = /) { 645 | return 302 {{ $location.Rewrite.AppRoot }}; 646 | } 647 | {{ end }} 648 | 649 | {{ if not (empty $authPath) }} 650 | location = {{ $authPath }} { 651 | internal; 652 | set $proxy_upstream_name "external-authentication"; 653 | 654 | proxy_pass_request_body off; 655 | proxy_set_header Content-Length ""; 656 | 657 | {{ if not (empty $location.ExternalAuth.Method) }} 658 | proxy_method {{ $location.ExternalAuth.Method }}; 659 | proxy_set_header X-Original-URI $request_uri; 660 | proxy_set_header X-Scheme $pass_access_scheme; 661 | {{ end }} 662 | 663 | proxy_set_header Host {{ $location.ExternalAuth.Host }}; 664 | proxy_set_header X-Original-URL $scheme://$http_host$request_uri; 665 | proxy_set_header X-Original-Method $request_method; 666 | proxy_set_header X-Auth-Request-Redirect $request_uri; 667 | proxy_set_header X-Sent-From "nginx-ingress-controller"; 668 | 669 | proxy_http_version 1.1; 670 | proxy_ssl_server_name on; 671 | proxy_pass_request_headers on; 672 | client_max_body_size "{{ $location.Proxy.BodySize }}"; 673 | {{ if isValidClientBodyBufferSize $location.ClientBodyBufferSize }} 674 | client_body_buffer_size {{ $location.ClientBodyBufferSize }}; 675 | {{ end }} 676 | 677 | set $target {{ $location.ExternalAuth.URL }}; 678 | proxy_pass $target; 679 | } 680 | {{ end }} 681 | 682 | location {{ $path }} { 683 | {{ if $all.Cfg.EnableVtsStatus }}{{ if $location.VtsFilterKey }} vhost_traffic_status_filter_by_set_key {{ $location.VtsFilterKey }};{{ end }}{{ end }} 684 | 685 | set $proxy_upstream_name "{{ buildUpstreamName $server.Hostname $all.Backends $location }}"; 686 | 687 | {{ $ing := (getIngressInformation $location.Ingress $path) }} 688 | {{/* $ing.Metadata contains the Ingress metadata */}} 689 | set $namespace "{{ $ing.Namespace }}"; 690 | set $ingress_name "{{ $ing.Rule }}"; 691 | set $service_name "{{ $ing.Service }}"; 692 | 693 | {{ if (or $location.Rewrite.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Rewrite.SSLRedirect)) }} 694 | # enforce ssl on server side 695 | if ($pass_access_scheme = http) { 696 | {{ if ne $all.ListenPorts.HTTPS 443 }} 697 | {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} 698 | return {{ $all.Cfg.HTTPRedirectCode }} https://$best_http_host{{ $redirect_port }}$request_uri; 699 | {{ else }} 700 | return {{ $all.Cfg.HTTPRedirectCode }} https://$best_http_host$request_uri; 701 | {{ end }} 702 | } 703 | {{ end }} 704 | 705 | {{ if $all.Cfg.EnableModsecurity }} 706 | modsecurity on; 707 | 708 | modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; 709 | {{ if $all.Cfg.EnableOWASPCoreRules }} 710 | modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; 711 | {{ end }} 712 | {{ end }} 713 | 714 | {{ if isLocationAllowed $location }} 715 | {{ if gt (len $location.Whitelist.CIDR) 0 }} 716 | if ({{ buildDenyVariable (print $server.Hostname "_" $path) }}) { 717 | return 403; 718 | } 719 | {{ end }} 720 | 721 | port_in_redirect {{ if $location.UsePortInRedirects }}on{{ else }}off{{ end }}; 722 | 723 | {{ if not (empty $authPath) }} 724 | # this location requires authentication 725 | auth_request {{ $authPath }}; 726 | auth_request_set $auth_cookie $upstream_http_set_cookie; 727 | add_header Set-Cookie $auth_cookie; 728 | {{- range $idx, $line := buildAuthResponseHeaders $location }} 729 | {{ $line }} 730 | {{- end }} 731 | {{ end }} 732 | 733 | {{ if not (empty $location.ExternalAuth.SigninURL) }} 734 | error_page 401 = {{ buildAuthSignURL $location.ExternalAuth.SigninURL }}; 735 | {{ end }} 736 | 737 | {{/* if the location contains a rate limit annotation, create one */}} 738 | {{ $limits := buildRateLimit $location }} 739 | {{ range $limit := $limits }} 740 | {{ $limit }}{{ end }} 741 | 742 | {{ if $location.BasicDigestAuth.Secured }} 743 | {{ if eq $location.BasicDigestAuth.Type "basic" }} 744 | auth_basic "{{ $location.BasicDigestAuth.Realm }}"; 745 | auth_basic_user_file {{ $location.BasicDigestAuth.File }}; 746 | {{ else }} 747 | auth_digest "{{ $location.BasicDigestAuth.Realm }}"; 748 | auth_digest_user_file {{ $location.BasicDigestAuth.File }}; 749 | {{ end }} 750 | proxy_set_header Authorization ""; 751 | {{ end }} 752 | 753 | {{ if $location.CorsConfig.CorsEnabled }} 754 | {{ template "CORS" $location }} 755 | {{ end }} 756 | 757 | {{ if not (empty $location.Redirect.URL) }} 758 | if ($uri ~* {{ $path }}) { 759 | return {{ $location.Redirect.Code }} {{ $location.Redirect.URL }}; 760 | } 761 | {{ end }} 762 | 763 | client_max_body_size "{{ $location.Proxy.BodySize }}"; 764 | {{ if isValidClientBodyBufferSize $location.ClientBodyBufferSize }} 765 | client_body_buffer_size {{ $location.ClientBodyBufferSize }}; 766 | {{ end }} 767 | 768 | {{/* By default use vhost as Host to upstream, but allow overrides */}} 769 | {{ if not (empty $location.UpstreamVhost) }} 770 | proxy_set_header Host "{{ $location.UpstreamVhost }}"; 771 | {{ else }} 772 | proxy_set_header Host $best_http_host; 773 | {{ end }} 774 | 775 | 776 | # Pass the extracted client certificate to the backend 777 | {{ if not (empty $server.CertificateAuth.CAFileName) }} 778 | {{ if $server.CertificateAuth.PassCertToUpstream }} 779 | proxy_set_header ssl-client-cert $ssl_client_escaped_cert; 780 | {{ else }} 781 | proxy_set_header ssl-client-cert ""; 782 | {{ end }} 783 | proxy_set_header ssl-client-verify $ssl_client_verify; 784 | proxy_set_header ssl-client-dn $ssl_client_s_dn; 785 | {{ else }} 786 | proxy_set_header ssl-client-cert ""; 787 | proxy_set_header ssl-client-verify ""; 788 | proxy_set_header ssl-client-dn ""; 789 | {{ end }} 790 | 791 | # Allow websocket connections 792 | proxy_set_header Upgrade $http_upgrade; 793 | proxy_set_header Connection $connection_upgrade; 794 | 795 | proxy_set_header X-Real-IP $the_real_ip; 796 | {{ if $all.Cfg.ComputeFullForwardedFor }} 797 | proxy_set_header X-Forwarded-For $full_x_forwarded_for; 798 | {{ else }} 799 | proxy_set_header X-Forwarded-For $the_real_ip; 800 | {{ end }} 801 | proxy_set_header X-Forwarded-Host $best_http_host; 802 | proxy_set_header X-Forwarded-Port $pass_port; 803 | proxy_set_header X-Forwarded-Proto $pass_access_scheme; 804 | proxy_set_header X-Original-URI $request_uri; 805 | proxy_set_header X-Scheme $pass_access_scheme; 806 | 807 | # Pass the original X-Forwarded-For 808 | proxy_set_header X-Original-Forwarded-For {{ buildForwardedFor $all.Cfg.ForwardedForHeader }}; 809 | 810 | # mitigate HTTPoxy Vulnerability 811 | # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ 812 | proxy_set_header Proxy ""; 813 | 814 | # Custom headers to proxied server 815 | {{ range $k, $v := $all.ProxySetHeaders }} 816 | proxy_set_header {{ $k }} "{{ $v }}"; 817 | {{ end }} 818 | 819 | proxy_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; 820 | proxy_send_timeout {{ $location.Proxy.SendTimeout }}s; 821 | proxy_read_timeout {{ $location.Proxy.ReadTimeout }}s; 822 | 823 | {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }} 824 | proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; 825 | {{ else }} 826 | proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; 827 | {{ end }} 828 | proxy_buffering off; 829 | proxy_buffer_size "{{ $location.Proxy.BufferSize }}"; 830 | proxy_buffers 4 "{{ $location.Proxy.BufferSize }}"; 831 | proxy_request_buffering "{{ $location.Proxy.RequestBuffering }}"; 832 | 833 | proxy_http_version 1.1; 834 | 835 | proxy_cookie_domain {{ $location.Proxy.CookieDomain }}; 836 | proxy_cookie_path {{ $location.Proxy.CookiePath }}; 837 | 838 | # In case of errors try the next upstream server before returning an error 839 | proxy_next_upstream {{ buildNextUpstream $location.Proxy.NextUpstream $all.Cfg.RetryNonIdempotent }}; 840 | 841 | {{/* rewrite only works if the content is not compressed */}} 842 | {{ if $location.Rewrite.AddBaseURL }} 843 | proxy_set_header Accept-Encoding ""; 844 | {{ end }} 845 | 846 | {{/* Add any additional configuration defined */}} 847 | {{ $location.ConfigurationSnippet }} 848 | 849 | {{ if not (empty $all.Cfg.LocationSnippet) }} 850 | # Custom code snippet configured in the configuration configmap 851 | {{ $all.Cfg.LocationSnippet }} 852 | {{ end }} 853 | 854 | {{/* if we are sending the request to a custom default backend, we add the required headers */}} 855 | {{ if (hasPrefix $location.Backend "custom-default-backend-") }} 856 | proxy_set_header X-Code 503; 857 | proxy_set_header X-Format $http_accept; 858 | proxy_set_header X-Namespace $namespace; 859 | proxy_set_header X-Ingress-Name $ingress_name; 860 | proxy_set_header X-Service-Name $service_name; 861 | {{ end }} 862 | 863 | 864 | {{ if not (empty $location.Backend) }} 865 | {{ buildProxyPass $server.Hostname $all.Backends $location }} 866 | {{ else }} 867 | # No endpoints available for the request 868 | return 503; 869 | {{ end }} 870 | {{ else }} 871 | # Location denied. Reason: {{ $location.Denied }} 872 | return 503; 873 | {{ end }} 874 | } 875 | 876 | {{ end }} 877 | 878 | {{ if eq $server.Hostname "_" }} 879 | # health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }} 880 | location {{ $all.HealthzURI }} { 881 | access_log off; 882 | return 200; 883 | } 884 | 885 | # this is required to avoid error if nginx is being monitored 886 | # with an external software (like sysdig) 887 | location /nginx_status { 888 | allow 127.0.0.1; 889 | {{ if $all.IsIPV6Enabled }}allow ::1;{{ end }} 890 | deny all; 891 | 892 | access_log off; 893 | stub_status on; 894 | } 895 | 896 | {{ end }} 897 | 898 | {{ end }} 899 | -------------------------------------------------------------------------------- /aws/test/k8s/ingress/nginx-ingress-controller.yaml: -------------------------------------------------------------------------------- 1 | # ServiceAccount for our Ingress controller 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: nginx-ingress-controller 6 | namespace: ingress 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1beta1 9 | kind: ClusterRole 10 | metadata: 11 | name: nginx-ingress-controller 12 | rules: 13 | - apiGroups: 14 | - "" 15 | resources: 16 | - configmaps 17 | - endpoints 18 | - nodes 19 | - pods 20 | - secrets 21 | verbs: 22 | - list 23 | - watch 24 | - apiGroups: 25 | - "" 26 | resources: 27 | - nodes 28 | verbs: 29 | - get 30 | - apiGroups: 31 | - "" 32 | resources: 33 | - services 34 | verbs: 35 | - get 36 | - list 37 | - watch 38 | - apiGroups: 39 | - "extensions" 40 | resources: 41 | - ingresses 42 | verbs: 43 | - get 44 | - list 45 | - watch 46 | - apiGroups: 47 | - "" 48 | resources: 49 | - events 50 | verbs: 51 | - create 52 | - patch 53 | - apiGroups: 54 | - "extensions" 55 | resources: 56 | - ingresses/status 57 | verbs: 58 | - update 59 | --- 60 | apiVersion: rbac.authorization.k8s.io/v1beta1 61 | kind: ClusterRoleBinding 62 | metadata: 63 | labels: 64 | k8s-addon: ingress-nginx.addons.k8s.io 65 | name: nginx-ingress-controller 66 | namespace: ingress 67 | roleRef: 68 | apiGroup: rbac.authorization.k8s.io 69 | kind: ClusterRole 70 | name: nginx-ingress-controller 71 | subjects: 72 | - apiGroup: rbac.authorization.k8s.io 73 | kind: User 74 | name: system:serviceaccount:ingress:nginx-ingress-controller 75 | --- 76 | apiVersion: rbac.authorization.k8s.io/v1beta1 77 | kind: Role 78 | metadata: 79 | name: nginx-ingress-controller 80 | namespace: ingress 81 | rules: 82 | - apiGroups: 83 | - "" 84 | resources: 85 | - configmaps 86 | - pods 87 | - secrets 88 | verbs: 89 | - get 90 | - apiGroups: 91 | - "" 92 | resources: 93 | - configmaps 94 | resourceNames: 95 | # Defaults to "-" 96 | # Here: "-" 97 | # This has to be adapted if you change either parameter 98 | # when launching the nginx-ingress-controller. 99 | - "ingress-controller-leader-nginx" 100 | verbs: 101 | - get 102 | - update 103 | - apiGroups: 104 | - "" 105 | resources: 106 | - configmaps 107 | verbs: 108 | - create 109 | - apiGroups: 110 | - "" 111 | resources: 112 | - endpoints 113 | verbs: 114 | - get 115 | - create 116 | - update 117 | --- 118 | apiVersion: rbac.authorization.k8s.io/v1beta1 119 | kind: RoleBinding 120 | metadata: 121 | labels: 122 | k8s-addon: ingress-nginx.addons.k8s.io 123 | name: nginx-ingress-controller 124 | namespace: ingress 125 | roleRef: 126 | apiGroup: rbac.authorization.k8s.io 127 | kind: Role 128 | name: nginx-ingress-controller 129 | subjects: 130 | - kind: ServiceAccount 131 | name: nginx-ingress-controller 132 | namespace: ingress 133 | --- 134 | # Ingress Controller config 135 | kind: ConfigMap 136 | apiVersion: v1 137 | metadata: 138 | name: nginx-ingress-controller 139 | namespace: ingress 140 | data: 141 | # Feature gates 142 | enable-vts-status: "true" 143 | enable-brotli: "false" 144 | use-gzip: "false" 145 | use-http2: "false" 146 | use-proxy-protocol: "true" 147 | 148 | # Proxy parameters 149 | worker-processes: "auto" 150 | max-worker-connections: "10000" 151 | 152 | proxy-body-size: "0" 153 | proxy-connect-timeout: "15" 154 | proxy-read-timeout: "600" 155 | proxy-send-timeout: "60" 156 | 157 | ## Retry behaviour (can be changed depending on how you handle the performance 158 | ## vs robustness tradeoff: if proxy request buffering is enabled, client 159 | ## requests will be written to RAM/disk before they are sent to the client 160 | ## which makes it possible to retry the request against a second backend in 161 | ## case a timeout/5xx error occurs... but this obviously requires extra 162 | ## RAM/disk space) 163 | proxy-request-buffering: "off" 164 | upstream-fail-timeout: "5" 165 | upstream-max-fails: "3" 166 | 167 | # TLS parameters 168 | ssl-protocols: "TLSv1 TLSv1.1 TLSv1.2" 169 | ssl-redirect: "true" 170 | hsts: "true" 171 | hsts-max-age: "100000" 172 | hsts-include-subdomains: "false" 173 | ssl-ciphers: "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" 174 | 175 | log-format-upstream: '{ "time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x-forward-for": "$proxy_add_x_forwarded_for", "request_id": "$request_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent" }' 176 | --- 177 | apiVersion: policy/v1beta1 178 | kind: PodDisruptionBudget 179 | metadata: 180 | name: ingress-nginx 181 | namespace: ingress 182 | spec: 183 | minAvailable: 1 184 | selector: 185 | matchLabels: 186 | k8s-app: nginx-ingress-controller 187 | --- 188 | apiVersion: autoscaling/v1 189 | kind: HorizontalPodAutoscaler 190 | metadata: 191 | name: ingress-nginx 192 | namespace: ingress 193 | spec: 194 | scaleTargetRef: 195 | apiVersion: extensions/v1beta1 196 | kind: Deployment 197 | name: ingress-nginx 198 | minReplicas: 1 199 | maxReplicas: 5 200 | targetCPUUtilizationPercentage: 30 201 | --- 202 | kind: Service 203 | apiVersion: v1 204 | metadata: 205 | name: ingress-nginx 206 | namespace: ingress 207 | labels: 208 | k8s-app: nginx-ingress-controller 209 | annotations: 210 | # Enable PROXY protocol 211 | service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: '*' 212 | # Increase the ELB idle timeout to avoid issues with WebSockets or Server-Sent Events. 213 | service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' 214 | # Enable cross-zone load-balancing 215 | service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true' 216 | spec: 217 | type: LoadBalancer 218 | selector: 219 | k8s-app: nginx-ingress-controller 220 | ports: 221 | - name: http 222 | port: 80 223 | targetPort: http 224 | - name: https 225 | port: 443 226 | targetPort: https 227 | --- 228 | # Ingress controller deployment 229 | kind: Deployment 230 | apiVersion: extensions/v1beta1 231 | metadata: 232 | name: ingress-nginx 233 | namespace: ingress 234 | labels: 235 | k8s-app: nginx-ingress-controller 236 | k8s-addon: ingress-nginx.addons.k8s.io 237 | karch-test: sure 238 | spec: 239 | strategy: 240 | type: RollingUpdate 241 | rollingUpdate: 242 | maxUnavailable: 1 243 | maxSurge: 0 244 | template: 245 | metadata: 246 | labels: 247 | app: ingress-nginx 248 | k8s-app: nginx-ingress-controller 249 | k8s-addon: ingress-nginx.addons.k8s.io 250 | annotations: 251 | prometheus.io/port: '10254' 252 | prometheus.io/scrape: 'true' 253 | spec: 254 | nodeSelector: 255 | duty: intake 256 | affinity: 257 | podAntiAffinity: 258 | requiredDuringSchedulingIgnoredDuringExecution: 259 | - labelSelector: 260 | matchExpressions: 261 | - key: k8s-app 262 | operator: In 263 | values: 264 | - nginx-ingress-controller 265 | topologyKey: kubernetes.io/hostname 266 | terminationGracePeriodSeconds: 10 267 | serviceAccountName: nginx-ingress-controller 268 | volumes: 269 | - name: nginx-conf-template 270 | configMap: 271 | name: nginx-conf-template 272 | containers: 273 | - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 274 | name: nginx-ingress-controller 275 | imagePullPolicy: Always 276 | volumeMounts: 277 | - name: nginx-conf-template 278 | mountPath: /etc/nginx/template 279 | readOnly: true 280 | ports: 281 | - name: http 282 | containerPort: 80 283 | protocol: TCP 284 | - name: https 285 | containerPort: 443 286 | protocol: TCP 287 | - name: status 288 | containerPort: 18080 289 | protocol: TCP 290 | readinessProbe: 291 | httpGet: 292 | path: /healthz 293 | port: 10254 294 | scheme: HTTP 295 | initialDelaySeconds: 1 296 | timeoutSeconds: 5 297 | livenessProbe: 298 | httpGet: 299 | path: /healthz 300 | port: 10254 301 | scheme: HTTP 302 | initialDelaySeconds: 10 303 | timeoutSeconds: 5 304 | resources: 305 | limits: 306 | memory: 400Mi 307 | requests: 308 | cpu: 500m 309 | memory: 400Mi 310 | env: 311 | - name: POD_NAME 312 | valueFrom: 313 | fieldRef: 314 | fieldPath: metadata.name 315 | - name: POD_NAMESPACE 316 | valueFrom: 317 | fieldRef: 318 | fieldPath: metadata.namespace 319 | args: 320 | - /nginx-ingress-controller 321 | - --default-backend-service=$(POD_NAMESPACE)/default-backend 322 | - --configmap=$(POD_NAMESPACE)/nginx-ingress-controller 323 | - --publish-service=$(POD_NAMESPACE)/ingress-nginx 324 | - --sort-backends 325 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/cluster-autoscaler.yaml: -------------------------------------------------------------------------------- 1 | # Service account and roles 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | k8s-addon: cluster-autoscaler.addons.k8s.io 7 | k8s-app: cluster-autoscaler 8 | name: cluster-autoscaler 9 | namespace: kube-system 10 | --- 11 | apiVersion: rbac.authorization.k8s.io/v1beta1 12 | kind: ClusterRole 13 | metadata: 14 | name: cluster-autoscaler 15 | labels: 16 | k8s-addon: cluster-autoscaler.addons.k8s.io 17 | k8s-app: cluster-autoscaler 18 | rules: 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - events 23 | - endpoints 24 | verbs: 25 | - create 26 | - patch 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - pods/eviction 31 | verbs: 32 | - create 33 | - apiGroups: 34 | - "" 35 | resources: 36 | - pods/status 37 | verbs: 38 | - update 39 | - apiGroups: 40 | - "" 41 | resources: 42 | - endpoints 43 | resourceNames: 44 | - cluster-autoscaler 45 | verbs: 46 | - get 47 | - update 48 | - apiGroups: 49 | - "" 50 | resources: 51 | - nodes 52 | verbs: 53 | - watch 54 | - list 55 | - get 56 | - update 57 | - apiGroups: 58 | - "" 59 | resources: 60 | - pods 61 | - services 62 | - replicationcontrollers 63 | - persistentvolumeclaims 64 | - persistentvolumes 65 | verbs: 66 | - watch 67 | - list 68 | - get 69 | - apiGroups: 70 | - extensions 71 | resources: 72 | - replicasets 73 | - daemonsets 74 | - deployments 75 | verbs: 76 | - watch 77 | - list 78 | - get 79 | - apiGroups: 80 | - policy 81 | resources: 82 | - poddisruptionbudgets 83 | verbs: 84 | - watch 85 | - list 86 | - apiGroups: 87 | - apps 88 | resources: 89 | - statefulsets 90 | verbs: 91 | - watch 92 | - list 93 | - get 94 | 95 | --- 96 | apiVersion: rbac.authorization.k8s.io/v1beta1 97 | kind: ClusterRoleBinding 98 | metadata: 99 | name: cluster-autoscaler 100 | labels: 101 | k8s-addon: cluster-autoscaler.addons.k8s.io 102 | k8s-app: cluster-autoscaler 103 | roleRef: 104 | apiGroup: rbac.authorization.k8s.io 105 | kind: ClusterRole 106 | name: cluster-autoscaler 107 | subjects: 108 | - kind: ServiceAccount 109 | name: cluster-autoscaler 110 | namespace: kube-system 111 | --- 112 | apiVersion: rbac.authorization.k8s.io/v1beta1 113 | kind: Role 114 | metadata: 115 | name: cluster-autoscaler 116 | namespace: kube-system 117 | labels: 118 | k8s-addon: cluster-autoscaler.addons.k8s.io 119 | k8s-app: cluster-autoscaler 120 | rules: 121 | - apiGroups: 122 | - "" 123 | resources: 124 | - configmaps 125 | verbs: 126 | - create 127 | - apiGroups: 128 | - "" 129 | resources: 130 | - configmaps 131 | resourceNames: 132 | - cluster-autoscaler-status 133 | verbs: 134 | - delete 135 | - get 136 | - update 137 | --- 138 | apiVersion: rbac.authorization.k8s.io/v1beta1 139 | kind: RoleBinding 140 | metadata: 141 | name: cluster-autoscaler 142 | namespace: kube-system 143 | labels: 144 | k8s-addon: cluster-autoscaler.addons.k8s.io 145 | k8s-app: cluster-autoscaler 146 | roleRef: 147 | apiGroup: rbac.authorization.k8s.io 148 | kind: Role 149 | name: cluster-autoscaler 150 | subjects: 151 | - kind: ServiceAccount 152 | name: cluster-autoscaler 153 | namespace: kube-system 154 | --- 155 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 156 | # node which has the cluster autoscaler on 157 | apiVersion: policy/v1beta1 158 | kind: PodDisruptionBudget 159 | metadata: 160 | name: cluster-autoscaler 161 | namespace: kube-system 162 | spec: 163 | minAvailable: 0 164 | selector: 165 | matchLabels: 166 | k8s-app: cluster-autoscaler 167 | --- 168 | # Actual cluster autoscaler deployment 169 | apiVersion: extensions/v1beta1 170 | kind: Deployment 171 | metadata: 172 | name: cluster-autoscaler 173 | namespace: kube-system 174 | labels: 175 | k8s-addon: cluster-autoscaler.addons.k8s.io 176 | k8s-app: cluster-autoscaler 177 | spec: 178 | replicas: 1 179 | strategy: 180 | type: RollingUpdate 181 | rollingUpdate: 182 | maxUnavailable: 0 183 | maxSurge: 1 184 | selector: 185 | matchLabels: 186 | k8s-app: cluster-autoscaler 187 | template: 188 | metadata: 189 | labels: 190 | k8s-addon: cluster-autoscaler.addons.k8s.io 191 | k8s-app: cluster-autoscaler 192 | spec: 193 | serviceAccountName: cluster-autoscaler 194 | nodeSelector: 195 | duty: webserver 196 | containers: 197 | - name: cluster-autoscaler 198 | image: gcr.io/google-containers/cluster-autoscaler:v1.0.3 199 | resources: 200 | limits: 201 | cpu: 100m 202 | memory: 300Mi 203 | requests: 204 | cpu: 100m 205 | memory: 300Mi 206 | command: 207 | - ./cluster-autoscaler 208 | - --cloud-provider=aws 209 | - --scale-down-enabled=true 210 | - --scale-down-delay-after-add=1m 211 | - --scale-down-utilization-threshold=0.6 212 | - --scale-down-unneeded-time=1m 213 | - --scale-down-unready-time=6m 214 | - --expander=least-waste 215 | - --stderrthreshold=warning 216 | - --v=4 217 | - --skip-nodes-with-local-storage=false 218 | # Per-ASG configuration 219 | - --nodes=1:5:ingress.test.morpheo.io 220 | - --nodes=1:3:default.test.morpheo.io 221 | env: 222 | - name: AWS_REGION 223 | value: eu-west-1 224 | volumeMounts: 225 | - name: ssl-certs 226 | mountPath: "/etc/ssl/certs/ca-certificates.crt" 227 | readOnly: true 228 | volumes: 229 | - name: ssl-certs 230 | hostPath: 231 | path: "/etc/ssl/certs/ca-certificates.crt" 232 | nodeSelector: 233 | node-role.kubernetes.io/master: "" 234 | tolerations: 235 | - key: "node-role.kubernetes.io/master" 236 | effect: NoSchedule 237 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/critical-addons-rescheduler.yaml: -------------------------------------------------------------------------------- 1 | # Service account and roles 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | k8s-addon: rescheduler.addons.k8s.io 7 | k8s-app: rescheduler 8 | name: rescheduler 9 | namespace: kube-system 10 | --- 11 | apiVersion: rbac.authorization.k8s.io/v1beta1 12 | kind: ClusterRole 13 | metadata: 14 | name: rescheduler 15 | labels: 16 | k8s-addon: rescheduler.addons.k8s.io 17 | k8s-app: rescheduler 18 | rules: 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - pods 23 | - nodes 24 | verbs: 25 | - list 26 | - watch 27 | - create 28 | - update 29 | --- 30 | apiVersion: rbac.authorization.k8s.io/v1beta1 31 | kind: ClusterRoleBinding 32 | metadata: 33 | name: rescheduler 34 | labels: 35 | k8s-addon: rescheduler.addons.k8s.io 36 | k8s-app: rescheduler 37 | roleRef: 38 | apiGroup: rbac.authorization.k8s.io 39 | kind: ClusterRole 40 | name: rescheduler 41 | subjects: 42 | - kind: ServiceAccount 43 | name: rescheduler 44 | namespace: kube-system 45 | --- 46 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 47 | # node which has the rescheduler on it 48 | apiVersion: policy/v1beta1 49 | kind: PodDisruptionBudget 50 | metadata: 51 | name: rescheduler 52 | namespace: kube-system 53 | spec: 54 | minAvailable: 0 55 | selector: 56 | matchLabels: 57 | app: rescheduler 58 | --- 59 | # Rescheduler 60 | # The code is short and simple and can be found here, better than any doc :) 61 | # https://github.com/kubernetes/contrib/blob/master/rescheduler/rescheduler.go 62 | apiVersion: extensions/v1beta1 63 | kind: Deployment 64 | metadata: 65 | name: rescheduler 66 | namespace: kube-system 67 | labels: 68 | app: rescheduler 69 | kubernetes.io/cluster-service: "true" 70 | spec: 71 | replicas: 1 72 | revisionHistoryLimit: 10 73 | selector: 74 | matchLabels: 75 | app: rescheduler 76 | kubernetes.io/cluster-service: "true" 77 | template: 78 | metadata: 79 | name: rescheduler 80 | namespace: kube-system 81 | labels: 82 | app: rescheduler 83 | version: v0.3.1 84 | kubernetes.io/cluster-service: "true" 85 | kubernetes.io/name: "Rescheduler" 86 | spec: 87 | serviceAccountName: rescheduler 88 | hostNetwork: true 89 | containers: 90 | - image: gcr.io/google_containers/rescheduler:v0.3.1 91 | name: rescheduler 92 | resources: 93 | limits: 94 | memory: 100Mi 95 | requests: 96 | cpu: 10m 97 | memory: 100Mi 98 | command: 99 | - '/rescheduler' 100 | args: 101 | - '--running-in-cluster=true' 102 | nodeSelector: 103 | duty: webserver 104 | restartPolicy: Always 105 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/dashboard.yaml: -------------------------------------------------------------------------------- 1 | # ServiceAccount & associated roles 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | k8s-app: kubernetes-dashboard 7 | name: kubernetes-dashboard 8 | namespace: kube-system 9 | --- 10 | apiVersion: rbac.authorization.k8s.io/v1beta1 11 | kind: ClusterRoleBinding 12 | metadata: 13 | name: kubernetes-dashboard 14 | labels: 15 | k8s-app: kubernetes-dashboard 16 | roleRef: 17 | apiGroup: rbac.authorization.k8s.io 18 | kind: ClusterRole 19 | name: cluster-admin 20 | subjects: 21 | - kind: ServiceAccount 22 | name: kubernetes-dashboard 23 | namespace: kube-system 24 | --- 25 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 26 | # node which has the kubernetes dashboard on it 27 | apiVersion: policy/v1beta1 28 | kind: PodDisruptionBudget 29 | metadata: 30 | name: kubernetes-dashboard 31 | namespace: kube-system 32 | spec: 33 | minAvailable: 0 34 | selector: 35 | matchLabels: 36 | k8s-app: kubernetes-dashboard 37 | # Actual service & deployment 38 | --- 39 | kind: Service 40 | apiVersion: v1 41 | metadata: 42 | labels: 43 | k8s-app: kubernetes-dashboard 44 | name: kubernetes-dashboard 45 | namespace: kube-system 46 | spec: 47 | ports: 48 | - port: 80 49 | targetPort: 9090 50 | selector: 51 | k8s-app: kubernetes-dashboard 52 | --- 53 | kind: Deployment 54 | apiVersion: extensions/v1beta1 55 | metadata: 56 | labels: 57 | k8s-app: kubernetes-dashboard 58 | name: kubernetes-dashboard 59 | namespace: kube-system 60 | spec: 61 | replicas: 1 62 | revisionHistoryLimit: 10 63 | selector: 64 | matchLabels: 65 | k8s-app: kubernetes-dashboard 66 | template: 67 | metadata: 68 | labels: 69 | k8s-app: kubernetes-dashboard 70 | spec: 71 | nodeSelector: 72 | duty: webserver 73 | containers: 74 | - name: kubernetes-dashboard 75 | image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.8.0 76 | ports: 77 | - containerPort: 9090 78 | protocol: TCP 79 | livenessProbe: 80 | httpGet: 81 | path: / 82 | port: 9090 83 | initialDelaySeconds: 30 84 | timeoutSeconds: 30 85 | serviceAccountName: kubernetes-dashboard 86 | # Comment the following tolerations if Dashboard must not be deployed on master 87 | tolerations: 88 | - key: node-role.kubernetes.io/master 89 | effect: NoSchedule 90 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/heapster.yaml: -------------------------------------------------------------------------------- 1 | # Service account & associated roles 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: heapster 6 | namespace: kube-system 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1beta1 9 | kind: ClusterRoleBinding 10 | metadata: 11 | name: heapster 12 | roleRef: 13 | apiGroup: rbac.authorization.k8s.io 14 | kind: ClusterRole 15 | name: system:heapster 16 | subjects: 17 | - kind: ServiceAccount 18 | name: heapster 19 | namespace: kube-system 20 | --- 21 | # Heapster's pod_nanny monitors the heapster deployment & its pod(s), and scales 22 | # the resources of the deployment if necessary. 23 | apiVersion: rbac.authorization.k8s.io/v1beta1 24 | kind: Role 25 | metadata: 26 | name: system:pod-nanny 27 | namespace: kube-system 28 | labels: 29 | kubernetes.io/cluster-service: "true" 30 | addonmanager.kubernetes.io/mode: Reconcile 31 | rules: 32 | - apiGroups: 33 | - "" 34 | resources: 35 | - pods 36 | verbs: 37 | - get 38 | - apiGroups: 39 | - "extensions" 40 | resources: 41 | - deployments 42 | verbs: 43 | - get 44 | - update 45 | --- 46 | apiVersion: rbac.authorization.k8s.io/v1beta1 47 | kind: RoleBinding 48 | metadata: 49 | name: heapster-binding 50 | namespace: kube-system 51 | labels: 52 | kubernetes.io/cluster-service: "true" 53 | addonmanager.kubernetes.io/mode: Reconcile 54 | roleRef: 55 | apiGroup: rbac.authorization.k8s.io 56 | kind: Role 57 | name: system:pod-nanny 58 | subjects: 59 | - kind: ServiceAccount 60 | name: heapster 61 | namespace: kube-system 62 | --- 63 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 64 | # node which has heapster on it 65 | apiVersion: policy/v1beta1 66 | kind: PodDisruptionBudget 67 | metadata: 68 | name: heapster 69 | namespace: kube-system 70 | spec: 71 | minAvailable: 0 72 | selector: 73 | matchLabels: 74 | k8s-app: heapster 75 | --- 76 | # Actual service & deployment 77 | apiVersion: v1 78 | kind: Service 79 | metadata: 80 | name: heapster 81 | namespace: kube-system 82 | labels: 83 | kubernetes.io/name: "Heapster" 84 | kubernetes.io/cluster-service: "true" 85 | spec: 86 | ports: 87 | - port: 80 88 | targetPort: 8082 89 | selector: 90 | k8s-app: heapster 91 | --- 92 | apiVersion: extensions/v1beta1 93 | kind: Deployment 94 | metadata: 95 | name: heapster 96 | namespace: kube-system 97 | labels: 98 | k8s-addon: monitoring-standalone.addons.k8s.io 99 | k8s-app: heapster 100 | kubernetes.io/cluster-service: "true" 101 | addonmanager.kubernetes.io/mode: Reconcile 102 | version: v1.7.0 103 | spec: 104 | replicas: 1 105 | selector: 106 | matchLabels: 107 | k8s-app: heapster 108 | version: v1.7.0 109 | template: 110 | metadata: 111 | labels: 112 | k8s-app: heapster 113 | version: v1.7.0 114 | annotations: 115 | scheduler.alpha.kubernetes.io/critical-pod: '' 116 | spec: 117 | serviceAccountName: heapster 118 | nodeSelector: 119 | duty: webserver 120 | containers: 121 | - image: gcr.io/google_containers/heapster:v1.4.0 122 | name: heapster 123 | livenessProbe: 124 | httpGet: 125 | path: /healthz 126 | port: 8082 127 | scheme: HTTP 128 | initialDelaySeconds: 180 129 | timeoutSeconds: 5 130 | resources: 131 | # keep request = limit to keep this container in guaranteed class 132 | limits: 133 | cpu: 100m 134 | memory: 300Mi 135 | requests: 136 | cpu: 100m 137 | memory: 300Mi 138 | command: 139 | - /heapster 140 | - --source=kubernetes.summary_api:'' 141 | - image: gcr.io/google_containers/addon-resizer:2.0 142 | name: heapster-nanny 143 | resources: 144 | limits: 145 | cpu: 50m 146 | memory: 100Mi 147 | requests: 148 | cpu: 50m 149 | memory: 100Mi 150 | env: 151 | - name: MY_POD_NAME 152 | valueFrom: 153 | fieldRef: 154 | fieldPath: metadata.name 155 | - name: MY_POD_NAMESPACE 156 | valueFrom: 157 | fieldRef: 158 | fieldPath: metadata.namespace 159 | command: 160 | - /pod_nanny 161 | - --cpu=80m 162 | - --extra-cpu=0.5m 163 | - --memory=140Mi 164 | - --extra-memory=4Mi 165 | - --deployment=heapster 166 | - --container=heapster 167 | - --poll-period=300000 168 | tolerations: 169 | - key: "CriticalAddonsOnly" 170 | operator: "Exists" 171 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/kube-dns.yaml: -------------------------------------------------------------------------------- 1 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 2 | # node which has the kube-dns-autoscaler on it 3 | apiVersion: policy/v1beta1 4 | kind: PodDisruptionBudget 5 | metadata: 6 | name: kube-dns-autoscaler 7 | namespace: kube-system 8 | spec: 9 | minAvailable: 0 10 | selector: 11 | matchLabels: 12 | k8s-app: kube-dns-autoscaler 13 | --- 14 | # Same thing for kube-dns but let's make sure there's always a living kube-dns 15 | # pod at least, as everything relies on it 16 | apiVersion: policy/v1beta1 17 | kind: PodDisruptionBudget 18 | metadata: 19 | name: kube-dns 20 | namespace: kube-system 21 | spec: 22 | minAvailable: 1 23 | selector: 24 | matchLabels: 25 | k8s-app: kube-dns 26 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/kube-state-metrics.yaml: -------------------------------------------------------------------------------- 1 | # Access control lists 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: kube-state-metrics 6 | namespace: kube-system 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1beta1 9 | kind: ClusterRole 10 | metadata: 11 | name: kube-state-metrics 12 | rules: 13 | - apiGroups: [""] 14 | resources: 15 | - nodes 16 | - pods 17 | - services 18 | - resourcequotas 19 | - replicationcontrollers 20 | - limitranges 21 | - persistentvolumeclaims 22 | verbs: ["list", "watch"] 23 | - apiGroups: ["extensions"] 24 | resources: 25 | - daemonsets 26 | - deployments 27 | - replicasets 28 | verbs: ["list", "watch"] 29 | - apiGroups: ["apps"] 30 | resources: 31 | - statefulsets 32 | verbs: ["list", "watch"] 33 | - apiGroups: ["batch"] 34 | resources: 35 | - cronjobs 36 | - jobs 37 | verbs: ["list", "watch"] 38 | --- 39 | apiVersion: rbac.authorization.k8s.io/v1beta1 40 | kind: ClusterRoleBinding 41 | metadata: 42 | name: kube-state-metrics 43 | roleRef: 44 | apiGroup: rbac.authorization.k8s.io 45 | kind: ClusterRole 46 | name: kube-state-metrics 47 | subjects: 48 | - kind: ServiceAccount 49 | name: kube-state-metrics 50 | namespace: kube-system 51 | --- 52 | apiVersion: rbac.authorization.k8s.io/v1beta1 53 | kind: Role 54 | metadata: 55 | namespace: kube-system 56 | name: kube-state-metrics-resizer 57 | rules: 58 | - apiGroups: [""] 59 | resources: 60 | - pods 61 | verbs: ["get"] 62 | - apiGroups: ["extensions"] 63 | resources: 64 | - deployments 65 | resourceNames: ["kube-state-metrics"] 66 | verbs: ["get", "update"] 67 | --- 68 | apiVersion: rbac.authorization.k8s.io/v1beta1 69 | kind: RoleBinding 70 | metadata: 71 | name: kube-state-metrics 72 | namespace: kube-system 73 | roleRef: 74 | apiGroup: rbac.authorization.k8s.io 75 | kind: Role 76 | name: kube-state-metrics-resizer 77 | subjects: 78 | - kind: ServiceAccount 79 | name: kube-state-metrics 80 | namespace: kube-system 81 | --- 82 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 83 | # node which has kube-state-metrics on it 84 | apiVersion: policy/v1beta1 85 | kind: PodDisruptionBudget 86 | metadata: 87 | name: kube-state-metrics 88 | namespace: kube-system 89 | spec: 90 | minAvailable: 0 91 | selector: 92 | matchLabels: 93 | k8s-app: kube-state-metrics 94 | --- 95 | # Actual service and deployment 96 | apiVersion: v1 97 | kind: Service 98 | metadata: 99 | name: kube-state-metrics 100 | namespace: kube-system 101 | labels: 102 | k8s-app: kube-state-metrics 103 | annotations: 104 | prometheus.io/scrape: 'true' 105 | spec: 106 | ports: 107 | - name: http-metrics 108 | port: 8080 109 | targetPort: http-metrics 110 | protocol: TCP 111 | selector: 112 | k8s-app: kube-state-metrics 113 | --- 114 | apiVersion: extensions/v1beta1 115 | kind: Deployment 116 | metadata: 117 | name: kube-state-metrics 118 | namespace: kube-system 119 | spec: 120 | replicas: 1 121 | template: 122 | metadata: 123 | labels: 124 | k8s-app: kube-state-metrics 125 | spec: 126 | serviceAccountName: kube-state-metrics 127 | containers: 128 | - name: kube-state-metrics 129 | image: gcr.io/google_containers/kube-state-metrics:v1.1.0 130 | ports: 131 | - name: http-metrics 132 | containerPort: 8080 133 | readinessProbe: 134 | httpGet: 135 | path: /healthz 136 | port: 8080 137 | initialDelaySeconds: 5 138 | timeoutSeconds: 5 139 | resources: 140 | requests: 141 | memory: 100Mi 142 | cpu: 100m 143 | limits: 144 | memory: 200Mi 145 | cpu: 200m 146 | - name: addon-resizer 147 | image: gcr.io/google_containers/addon-resizer:1.0 148 | resources: 149 | limits: 150 | cpu: 100m 151 | memory: 30Mi 152 | requests: 153 | cpu: 100m 154 | memory: 30Mi 155 | env: 156 | - name: MY_POD_NAME 157 | valueFrom: 158 | fieldRef: 159 | fieldPath: metadata.name 160 | - name: MY_POD_NAMESPACE 161 | valueFrom: 162 | fieldRef: 163 | fieldPath: metadata.namespace 164 | command: 165 | - /pod_nanny 166 | - --container=kube-state-metrics 167 | - --cpu=100m 168 | - --extra-cpu=1m 169 | - --memory=100Mi 170 | - --extra-memory=2Mi 171 | - --threshold=5 172 | - --deployment=kube-state-metrics 173 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/npd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: node-problem-detector 5 | namespace: kube-system 6 | labels: 7 | kubernetes.io/cluster-service: "true" 8 | addonmanager.kubernetes.io/mode: Reconcile 9 | --- 10 | apiVersion: rbac.authorization.k8s.io/v1beta1 11 | kind: ClusterRoleBinding 12 | metadata: 13 | name: npd-binding 14 | labels: 15 | kubernetes.io/cluster-service: "true" 16 | addonmanager.kubernetes.io/mode: Reconcile 17 | roleRef: 18 | apiGroup: rbac.authorization.k8s.io 19 | kind: ClusterRole 20 | name: system:node-problem-detector 21 | subjects: 22 | - kind: ServiceAccount 23 | name: node-problem-detector 24 | namespace: kube-system 25 | --- 26 | apiVersion: extensions/v1beta1 27 | kind: DaemonSet 28 | metadata: 29 | name: npd-v0.4.1 30 | namespace: kube-system 31 | labels: 32 | k8s-app: node-problem-detector 33 | version: v0.4.1 34 | kubernetes.io/cluster-service: "true" 35 | addonmanager.kubernetes.io/mode: Reconcile 36 | spec: 37 | template: 38 | metadata: 39 | labels: 40 | k8s-app: node-problem-detector 41 | version: v0.4.1 42 | kubernetes.io/cluster-service: "true" 43 | spec: 44 | containers: 45 | - name: node-problem-detector 46 | image: gcr.io/google_containers/node-problem-detector:v0.4.1 47 | command: 48 | - "/bin/sh" 49 | - "-c" 50 | # Pass both config to support both journald and syslog. 51 | - "/node-problem-detector --logtostderr --system-log-monitors=/config/kernel-monitor.json,/config/kernel-monitor-filelog.json,/config/docker-monitor.json,/config/docker-monitor-filelog.json >>/var/log/node-problem-detector.log 2>&1" 52 | securityContext: 53 | privileged: true 54 | resources: 55 | limits: 56 | cpu: "200m" 57 | memory: "100Mi" 58 | requests: 59 | cpu: "20m" 60 | memory: "20Mi" 61 | env: 62 | - name: NODE_NAME 63 | valueFrom: 64 | fieldRef: 65 | fieldPath: spec.nodeName 66 | volumeMounts: 67 | - name: log 68 | mountPath: /var/log 69 | - name: localtime 70 | mountPath: /etc/localtime 71 | readOnly: true 72 | volumes: 73 | - name: log 74 | hostPath: 75 | path: /var/log/ 76 | - name: localtime 77 | hostPath: 78 | path: /etc/localtime 79 | serviceAccountName: node-problem-detector 80 | tolerations: 81 | - operator: "Exists" 82 | effect: "NoExecute" 83 | - key: "CriticalAddonsOnly" 84 | operator: "Exists" 85 | -------------------------------------------------------------------------------- /aws/test/k8s/namespaces.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: ingress 5 | -------------------------------------------------------------------------------- /aws/test/provider.tf: -------------------------------------------------------------------------------- 1 | // Default AWS provider 2 | provider "aws" { 3 | region = "eu-west-1" 4 | } 5 | -------------------------------------------------------------------------------- /aws/test/test-cluster.tf: -------------------------------------------------------------------------------- 1 | module "test-cluster" { 2 | source = "./test-cluster" 3 | 4 | aws-region = "${data.aws_region.main.name}" 5 | 6 | kubernetes-version = "${var.kubernetes-version}" 7 | 8 | # Networking 9 | vpc-name = "karch-test-1" 10 | vpc-number = "10.0.0.0/16" 11 | availability-zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 12 | trusted-cidrs = ["0.0.0.0/0"] 13 | admin-ssh-public-key-path = "~/.ssh/terraform.pub" 14 | 15 | # DNS 16 | main-zone-id = "${data.aws_route53_zone.test.id}" 17 | cluster-name = "test.${var.domain}" 18 | kube-dns-domain = "cluster.karch-test-1" 19 | 20 | # Kops & Kuberntetes 21 | kops-state-bucket = "${var.kops-state-bucket}" 22 | cloud-labels = "${map("karch-test", "true")}" 23 | base-ami = "${data.aws_ami.coreos-stable.id}" 24 | 25 | # Master 26 | master-availability-zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 27 | master-machine-type = "m4.large" 28 | master-volume-size = "30" 29 | master-volume-type = "gp2" 30 | 31 | # First minion instance group (HTTP webservers of all types + kube-system pods) 32 | cluster-base-minion-ig-name = "default" 33 | cluster-base-minion-machine-type = "m4.large" 34 | cluster-base-minions-min = 1 35 | cluster-base-minions-max = 3 36 | 37 | # Ingress nodes 38 | ingress-nodes-subnets = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 39 | ingress-machine-type = "m4.large" 40 | ingress-min-nodes = 1 41 | ingress-max-nodes = 5 42 | } 43 | -------------------------------------------------------------------------------- /aws/test/test-cluster/ingress.tf: -------------------------------------------------------------------------------- 1 | # Note: we should put the ingress controller on default nodes and deploy a single AZ database node group on which to 2 | # spawn statefulsets (maybe an EFK monitoring stack ?) so that we can check the successful creation of stateful sets and 3 | # - therefore - persistent volumes. 4 | module "ingress-ig" { 5 | source = "../../ig" 6 | 7 | # Master cluster dependency hook 8 | master-up = "${module.kops-cluster.master-up}" 9 | 10 | # Global config 11 | name = "ingress" 12 | cluster-name = "${var.cluster-name}" 13 | kops-state-bucket = "${var.kops-state-bucket}" 14 | visibility = "private" 15 | subnets = ["${var.ingress-nodes-subnets}"] 16 | image = "${var.base-ami}" 17 | type = "${var.ingress-machine-type}" 18 | volume-size = "30" 19 | volume-type = "gp2" 20 | min-size = "${var.ingress-min-nodes}" 21 | max-size = "${var.ingress-max-nodes}" 22 | node-labels = "${map("duty", "intake")}" 23 | 24 | # hooks = [ 25 | # <