├── aws ├── test │ ├── k8s │ │ ├── namespaces.yaml │ │ ├── kube-system │ │ │ ├── kube-dns.yaml │ │ │ ├── dashboard.yaml │ │ │ ├── npd.yaml │ │ │ ├── critical-addons-rescheduler.yaml │ │ │ ├── kube-state-metrics.yaml │ │ │ ├── heapster.yaml │ │ │ └── cluster-autoscaler.yaml │ │ ├── ingress │ │ │ ├── kube-lego.yaml │ │ │ ├── default-backend.yaml │ │ │ ├── nginx-ingress-controller.yaml │ │ │ └── nginx-ingress-controller-config.yaml │ │ └── backend │ │ │ └── pong.yaml │ ├── provider.tf │ ├── test-cluster │ │ ├── provider.tf │ │ ├── outputs.tf │ │ ├── ingress.tf │ │ ├── kops-cluster.tf │ │ ├── kernel-tuning.tf │ │ └── variables.tf │ ├── dns.tf │ ├── iam-kops.tf │ ├── variables.tf │ ├── data.tf │ └── test-cluster.tf ├── ig │ ├── outputs.tf │ ├── data.tf │ ├── templates │ │ └── spec.yaml │ ├── ig-spec-template.tf │ ├── main.tf │ └── variables.tf └── cluster │ ├── data.tf │ ├── templates │ ├── ig-spec.yaml │ └── cluster-spec.yaml │ ├── route53.tf │ ├── outputs.tf │ ├── networking.tf │ ├── main.tf │ ├── cluster-spec-template.tf │ ├── cluster-head-nodes-template.tf │ └── variables.tf ├── .gitignore ├── CHANGELOG.md ├── README.md └── LICENSE /aws/test/k8s/namespaces.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: ingress 5 | -------------------------------------------------------------------------------- /aws/test/provider.tf: -------------------------------------------------------------------------------- 1 | // Default AWS provider 2 | provider "aws" { 3 | region = "eu-west-1" 4 | } 5 | -------------------------------------------------------------------------------- /aws/ig/outputs.tf: -------------------------------------------------------------------------------- 1 | output "created" { 2 | value = null_resource.ig.id 3 | } 4 | 5 | // ID of the AWS autoscaling group corresponding to this kops instance group 6 | output "asg-name" { 7 | value = element(data.aws_autoscaling_groups.ig.names, 0) 8 | } 9 | -------------------------------------------------------------------------------- /aws/test/test-cluster/provider.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * AWS cloud provider & terraform user definition (spans the entire Morpheo Org.) 3 | */ 4 | 5 | // Default AWS provider, used for worldwide resources (IAM assets, main DNS zone...) 6 | provider "aws" { 7 | region = "${var.aws-region}" 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Terraform 2 | # Compiled files 3 | *.tfstate 4 | *.tfstate.backup 5 | 6 | # Module directory 7 | .terraform/ 8 | 9 | ## Vim 10 | # Swap 11 | [._]*.s[a-v][a-z] 12 | [._]*.sw[a-p] 13 | [._]s[a-v][a-z] 14 | [._]sw[a-p] 15 | 16 | # Session 17 | Session.vim 18 | 19 | # Temporary 20 | .netrwhist 21 | *~ 22 | # Auto-generated tag files 23 | tags 24 | -------------------------------------------------------------------------------- /aws/ig/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_autoscaling_groups" "ig" { 2 | filter { 3 | name = "auto-scaling-group" 4 | 5 | // The second value simply is a way to make sure this data source will 6 | // only be provisioned once these subnets have been created by kops 7 | values = ["${var.name}.${var.cluster-name}", null_resource.ig-update.id] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /aws/test/test-cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | output "subnets" { 2 | value = "${module.kops-cluster.subnets}" 3 | } 4 | 5 | output "nodes-sg" { 6 | value = "${module.kops-cluster.nodes-sg}" 7 | } 8 | 9 | output "vpc-id" { 10 | value = "${module.kops-cluster.vpc-id}" 11 | } 12 | 13 | output "master-up" { 14 | value = "${module.kops-cluster.master-up}" 15 | } 16 | -------------------------------------------------------------------------------- /aws/test/dns.tf: -------------------------------------------------------------------------------- 1 | resource "aws_route53_record" "pong" { 2 | zone_id = "${data.aws_route53_zone.test.id}" 3 | name = "pong" 4 | type = "A" 5 | 6 | alias { 7 | name = "${module.test-cluster.ingress-elb-dns-name}" 8 | zone_id = "${module.test-cluster.ingress-elb-zone-id}" 9 | evaluate_target_health = "false" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /aws/cluster/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_security_group" "nodes" { 2 | vpc_id = aws_vpc.main.id 3 | 4 | filter { 5 | name = "tag:Name" 6 | 7 | // The second value is just a hacky dependency hooks on our cluster being created 8 | values = ["nodes.${var.cluster-name}", null_resource.master-up.id] 9 | } 10 | } 11 | 12 | data "aws_security_group" "masters" { 13 | vpc_id = aws_vpc.main.id 14 | 15 | filter { 16 | name = "tag:Name" 17 | 18 | // The second value is just a hacky dependency hooks on our cluster being created 19 | values = ["masters.${var.cluster-name}", null_resource.master-up.id] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /aws/ig/templates/spec.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kops/v1alpha2 2 | kind: InstanceGroup 3 | metadata: 4 | labels: 5 | kops.k8s.io/cluster: ${cluster-name} 6 | name: ${name} 7 | spec: 8 | cloudLabels: 9 | ${cloud-labels} 10 | nodeLabels: 11 | ${node-labels} 12 | associatePublicIp: ${public} 13 | ${additional-sgs} 14 | image: ${image} 15 | machineType: ${type} 16 | maxSize: ${max-size} 17 | minSize: ${min-size} 18 | role: ${role} 19 | rootVolumeSize: ${volume-size} 20 | rootVolumeType: ${volume-type} 21 | rootProvisionedIops: ${volume-provisioned-iops} 22 | rootVolumeOptimization: ${ebs-optimized} 23 | ${max-price} 24 | taints: 25 | ${taints} 26 | subnets: 27 | ${subnets} 28 | hooks: 29 | ${hooks} 30 | -------------------------------------------------------------------------------- /aws/cluster/templates/ig-spec.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kops/v1alpha2 2 | kind: InstanceGroup 3 | metadata: 4 | labels: 5 | kops.k8s.io/cluster: ${cluster-name} 6 | name: ${name} 7 | spec: 8 | cloudLabels: 9 | ${cloud-labels} 10 | nodeLabels: 11 | ${node-labels} 12 | associatePublicIp: ${public} 13 | ${additional-sgs} 14 | image: ${image} 15 | machineType: ${type} 16 | maxSize: ${max-size} 17 | minSize: ${min-size} 18 | role: ${role} 19 | rootVolumeSize: ${volume-size} 20 | rootVolumeType: ${volume-type} 21 | rootProvisionedIops: ${volume-provisioned-iops} 22 | rootVolumeOptimization: ${ebs-optimized} 23 | ${max-price} 24 | taints: 25 | ${taints} 26 | subnets: 27 | ${subnets} 28 | hooks: 29 | ${hooks} 30 | -------------------------------------------------------------------------------- /aws/test/iam-kops.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * IAM policies for kops are defined here, according to the "kops-policies" variable 3 | */ 4 | 5 | # Kops 6 | resource "aws_iam_group" "kops" { 7 | name = "kops-test" 8 | path = "/" 9 | } 10 | 11 | resource "aws_iam_group_policy_attachment" "kops" { 12 | count = "${length(var.kops-policies)}" 13 | group = "${aws_iam_group.kops.name}" 14 | policy_arn = "arn:aws:iam::aws:policy/${element(var.kops-policies, count.index)}" 15 | } 16 | 17 | resource "aws_iam_group_membership" "kops" { 18 | name = "kops-membership" 19 | group = "${aws_iam_group.kops.name}" 20 | users = ["${aws_iam_user.kops.name}"] 21 | } 22 | 23 | resource "aws_iam_user" "kops" { 24 | name = "kops-test" 25 | path = "/" 26 | } 27 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/kube-dns.yaml: -------------------------------------------------------------------------------- 1 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 2 | # node which has the kube-dns-autoscaler on it 3 | apiVersion: policy/v1beta1 4 | kind: PodDisruptionBudget 5 | metadata: 6 | name: kube-dns-autoscaler 7 | namespace: kube-system 8 | spec: 9 | minAvailable: 0 10 | selector: 11 | matchLabels: 12 | k8s-app: kube-dns-autoscaler 13 | --- 14 | # Same thing for kube-dns but let's make sure there's always a living kube-dns 15 | # pod at least, as everything relies on it 16 | apiVersion: policy/v1beta1 17 | kind: PodDisruptionBudget 18 | metadata: 19 | name: kube-dns 20 | namespace: kube-system 21 | spec: 22 | minAvailable: 1 23 | selector: 24 | matchLabels: 25 | k8s-app: kube-dns 26 | -------------------------------------------------------------------------------- /aws/cluster/route53.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * Cluster public subdomain configuration 3 | */ 4 | 5 | resource "aws_route53_zone" "cluster" { 6 | name = var.cluster-name 7 | force_destroy = true 8 | 9 | vpc { 10 | vpc_id = aws_vpc.main.id 11 | } 12 | 13 | dynamic "vpc" { 14 | for_each = var.extra-route53-vpc-associations 15 | 16 | content { 17 | vpc_id = vpc.value 18 | } 19 | } 20 | } 21 | 22 | resource "aws_route53_record" "cluster-root" { 23 | count = var.master-lb-visibility == "Private" ? 0 : 1 24 | 25 | zone_id = var.main-zone-id 26 | name = var.cluster-name 27 | type = "NS" 28 | ttl = "30" 29 | 30 | records = [ 31 | aws_route53_zone.cluster.name_servers.0, 32 | aws_route53_zone.cluster.name_servers.1, 33 | aws_route53_zone.cluster.name_servers.2, 34 | aws_route53_zone.cluster.name_servers.3, 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /aws/test/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws-account-id" { 2 | type = "string" 3 | description = "AWS account ID for your org." 4 | } 5 | 6 | variable "kops-policies" { 7 | description = "IAM policies necessary for kops to work" 8 | type = "list" 9 | 10 | default = [ 11 | "AmazonEC2FullAccess", 12 | "AmazonRoute53FullAccess", 13 | "AmazonS3FullAccess", 14 | "IAMFullAccess", 15 | "AmazonVPCFullAccess", 16 | ] 17 | } 18 | 19 | variable "domain" { 20 | type = "string" 21 | description = "Main domain name for the organization (ex.: example.com)" 22 | } 23 | 24 | variable "kops-state-bucket" { 25 | description = "Name of the S3 bucket that will hold the Kops state files" 26 | type = "string" 27 | } 28 | 29 | variable "kubernetes-version" { 30 | type = "string" 31 | description = "Kubernetes version to use for Core components (default: v1.7.4)" 32 | } 33 | -------------------------------------------------------------------------------- /aws/cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | # Lifecycle hooks 2 | output "master-up" { 3 | value = null_resource.master-up.id 4 | } 5 | 6 | output "cluster-created" { 7 | value = null_resource.kops-cluster.id 8 | } 9 | 10 | # DNS zone for the cluster subdomain 11 | output "cluster-zone-id" { 12 | value = aws_route53_zone.cluster.id 13 | } 14 | 15 | output "vpc-id" { 16 | value = aws_vpc.main.id 17 | } 18 | 19 | output "cluster-cidr-block" { 20 | value = aws_vpc.main.cidr_block 21 | } 22 | 23 | // List of utility (public) subnets 24 | output "utility-subnets" { 25 | value = aws_subnet.utility.*.id 26 | } 27 | 28 | // Standard IG subnets 29 | output "subnets" { 30 | value = aws_subnet.private.*.id 31 | } 32 | 33 | // Utility (public) route tables 34 | output "utility-route-tables" { 35 | value = aws_route_table.utility.*.id 36 | } 37 | 38 | // Standard route tables 39 | output "route-tables" { 40 | value = aws_route_table.private.*.id 41 | } 42 | 43 | // Nodes security groups (to direct ELB traffic to hostPort pods if you want to) 44 | output "nodes-sg" { 45 | value = element(split("/", data.aws_security_group.nodes.arn), 1) 46 | } 47 | -------------------------------------------------------------------------------- /aws/test/data.tf: -------------------------------------------------------------------------------- 1 | # AWS account information 2 | data "aws_caller_identity" "current" {} 3 | 4 | data "aws_region" "main" { 5 | current = true 6 | } 7 | 8 | # CoreOS base AMI 9 | variable "coreos-ami-owner-id" { 10 | description = "The ID of the owner of the CoreOS image you want to use on the AWS marketplace (or yours if you're using your own AMI)." 11 | 12 | # CoreOS' official AWS id 13 | default = "595879546273" 14 | type = "string" 15 | } 16 | 17 | variable "coreos-ami-pattern" { 18 | description = "The AMI pattern to use (it can be a full name or contain wildcards, default to the last release of CoreOS on the stable channel)." 19 | 20 | # Useful to change that to also run tests against the beta and alpha version of Container Linux 21 | default = "CoreOS-stable-*" 22 | type = "string" 23 | } 24 | 25 | data "aws_ami" "coreos-stable" { 26 | most_recent = true 27 | 28 | filter { 29 | name = "name" 30 | values = ["${var.coreos-ami-pattern}"] 31 | } 32 | 33 | filter { 34 | name = "virtualization-type" 35 | values = ["hvm"] 36 | } 37 | 38 | owners = ["${var.coreos-ami-owner-id}"] 39 | } 40 | 41 | data "aws_route53_zone" "test" { 42 | name = "${var.domain}" 43 | private_zone = "false" 44 | } 45 | -------------------------------------------------------------------------------- /aws/test/test-cluster/ingress.tf: -------------------------------------------------------------------------------- 1 | # Note: we should put the ingress controller on default nodes and deploy a single AZ database node group on which to 2 | # spawn statefulsets (maybe an EFK monitoring stack ?) so that we can check the successful creation of stateful sets and 3 | # - therefore - persistent volumes. 4 | module "ingress-ig" { 5 | source = "../../ig" 6 | 7 | # Master cluster dependency hook 8 | master-up = "${module.kops-cluster.master-up}" 9 | 10 | # Global config 11 | name = "ingress" 12 | cluster-name = "${var.cluster-name}" 13 | kops-state-bucket = "${var.kops-state-bucket}" 14 | visibility = "private" 15 | subnets = ["${var.ingress-nodes-subnets}"] 16 | image = "${var.base-ami}" 17 | type = "${var.ingress-machine-type}" 18 | volume-size = "30" 19 | volume-type = "gp2" 20 | min-size = "${var.ingress-min-nodes}" 21 | max-size = "${var.ingress-max-nodes}" 22 | node-labels = "${map("duty", "intake")}" 23 | 24 | # hooks = [ 25 | # <>/var/log/node-problem-detector.log 2>&1" 52 | securityContext: 53 | privileged: true 54 | resources: 55 | limits: 56 | cpu: "200m" 57 | memory: "100Mi" 58 | requests: 59 | cpu: "20m" 60 | memory: "20Mi" 61 | env: 62 | - name: NODE_NAME 63 | valueFrom: 64 | fieldRef: 65 | fieldPath: spec.nodeName 66 | volumeMounts: 67 | - name: log 68 | mountPath: /var/log 69 | - name: localtime 70 | mountPath: /etc/localtime 71 | readOnly: true 72 | volumes: 73 | - name: log 74 | hostPath: 75 | path: /var/log/ 76 | - name: localtime 77 | hostPath: 78 | path: /etc/localtime 79 | serviceAccountName: node-problem-detector 80 | tolerations: 81 | - operator: "Exists" 82 | effect: "NoExecute" 83 | - key: "CriticalAddonsOnly" 84 | operator: "Exists" 85 | -------------------------------------------------------------------------------- /aws/ig/ig-spec-template.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "ig-spec" { 2 | template = file("${path.module}/templates/spec.yaml") 3 | 4 | vars = { 5 | cluster-name = var.cluster-name 6 | cloud-labels = join("\n", data.template_file.cloud-labels.*.rendered) 7 | node-labels = join("\n", data.template_file.node-labels.*.rendered) 8 | name = var.name 9 | public = var.visibility == "public" ? "true" : "false" 10 | 11 | additional-sgs = < 0 ? "additionalSecurityGroups:" : ""} 13 | ${join("\n", data.template_file.additional-sgs.*.rendered)} 14 | EOF 15 | 16 | image = var.image 17 | type = var.type 18 | max-size = var.max-size 19 | min-size = var.min-size 20 | role = "Node" 21 | volume-size = var.volume-size 22 | volume-provisioned-iops = var.volume-provisioned-iops == "" ? "" : var.volume-provisioned-iops 23 | volume-type = var.volume-type 24 | ebs-optimized = var.ebs-optimized 25 | max-price = "maxPrice: '${var.max-price}'" 26 | taints = join("\n", data.template_file.taints.*.rendered) 27 | subnets = join("\n", data.template_file.subnets.*.rendered) 28 | hooks = join("\n", data.template_file.hooks.*.rendered) 29 | } 30 | } 31 | 32 | data "template_file" "additional-sgs" { 33 | count = var.additional-sgs-count 34 | 35 | template = " - $${sg-id}" 36 | 37 | vars = { 38 | sg-id = element(var.additional-sgs, count.index) 39 | } 40 | } 41 | 42 | data "template_file" "cloud-labels" { 43 | count = length(keys(var.cloud-labels)) 44 | 45 | template = < ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 24 | ${aws_s3_bucket_object.ig-spec.content} 25 | EOF 26 | FILEDUMP 27 | } 28 | 29 | // Let's register our Kops cluster into remote state 30 | provisioner "local-exec" { 31 | command = < ${path.root}/.kops-ig-lock 38 | 39 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} create -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 40 | 41 | rmdir ${path.root}/.kops-ig-lock 42 | EOF 43 | } 44 | 45 | // Let's remove the ig spec from disk 46 | provisioner "local-exec" { 47 | command = "rm -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml" 48 | } 49 | 50 | depends_on = [aws_s3_bucket_object.ig-spec] 51 | } 52 | 53 | resource "null_resource" "ig-update" { 54 | triggers = { 55 | cluster_spec = data.template_file.ig-spec.rendered 56 | } 57 | 58 | provisioner "local-exec" { 59 | command = < ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 61 | ${data.template_file.ig-spec.rendered} 62 | EOF 63 | FILEDUMP 64 | } 65 | 66 | provisioner "local-exec" { 67 | command = < ${path.root}/.kops-ig-lock 74 | 75 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} \ 76 | replace -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 77 | 78 | rm -f ${path.module}/${var.cluster-name}-${var.name}-ig-spec.yml 79 | 80 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} \ 81 | update cluster ${var.cluster-name} --yes 82 | 83 | KOPS_FEATURE_FLAGS="+DrainAndValidateRollingUpdate" \ 84 | ${var.nodeup-url-env} AWS_SDK_LOAD_CONFIG=1 AWS_PROFILE=${var.aws-profile} kops --state=s3://${var.kops-state-bucket} \ 85 | rolling-update cluster ${var.cluster-name} \ 86 | --node-interval="${var.update-interval}m" ${var.automatic-rollout == "true" ? "--yes" : ""}\ 87 | --instance-group="${var.name}" 88 | 89 | rmdir ${path.root}/.kops-ig-lock 90 | EOF 91 | } 92 | 93 | depends_on = [null_resource.ig] 94 | } 95 | -------------------------------------------------------------------------------- /aws/test/test-cluster/kernel-tuning.tf: -------------------------------------------------------------------------------- 1 | # Kernel parameters to enforce on our nodes via a systemd hook 2 | data "template_file" "webserver-sysctl-parameters" { 3 | count = "${length(var.webserver-sysctl-parameters)}" 4 | 5 | template = " ExecStartPre=/usr/sbin/sysctl -w ${element(var.webserver-sysctl-parameters, count.index)}" 6 | } 7 | 8 | variable "webserver-sysctl-parameters" { 9 | type = "list" 10 | 11 | default = [ 12 | # Number of file descriptors on the system 13 | "fs.file-max=2097152", 14 | 15 | # Max number of orphaned TCP sockets 16 | "net.ipv4.tcp_max_orphans=60000", 17 | 18 | # Do not save metrics when connexion are closed to spare CPU&IO time 19 | "net.ipv4.tcp_no_metrics_save=1", 20 | 21 | # Enable TCP window scaling 22 | # https://en.wikipedia.org/wiki/TCP_window_scale_option 23 | "net.ipv4.tcp_window_scaling=1", 24 | 25 | # As per RFC 1323, enabling round trip measurements might feel like a good idea at the age of fiber optics 26 | "net.ipv4.tcp_timestamps=1", 27 | 28 | # Enable select acknowledgments: 29 | "net.ipv4.tcp_sack=1", 30 | 31 | // Allowed local port range 32 | "net.ipv4.ip_local_port_range=1024 65535", 33 | 34 | # Maximum number of remembered connection requests, which did not yet 35 | # receive an acknowledgment from connecting client. 36 | "net.ipv4.tcp_max_syn_backlog=10240", 37 | 38 | # Choose the HTCP congestion control algorithm 39 | "net.ipv4.tcp_congestion_control=htcp", 40 | 41 | # recommended for hosts with jumbo frames enabled 42 | "net.ipv4.tcp_mtu_probing=1", 43 | 44 | # Number of times SYNACKs for passive TCP connection. 45 | "net.ipv4.tcp_synack_retries=2", 46 | 47 | # Protect Against TCP Time-Wait 48 | "net.ipv4.tcp_rfc1337=1", 49 | 50 | # Decrease the time default value for tcp_fin_timeout connection 51 | "net.ipv4.tcp_fin_timeout=15", 52 | 53 | # Increase number of incoming connections 54 | # somaxconn defines the number of request_sock structures 55 | # allocated per each listen call. The 56 | # queue is persistent through the life of the listen socket. 57 | "net.core.somaxconn=1024", // or more ? 58 | 59 | # Increase number of incoming connections backlog queue 60 | # Sets the maximum number of packets, queued on the INPUT 61 | # side, when the interface receives packets faster than 62 | # kernel can process them. 63 | "net.core.netdev_max_backlog=65536", // or more ? 64 | 65 | # Increase the maximum amount of option memory buffers 66 | "net.core.optmem_max=25165824", 67 | 68 | # Increase the maximum total buffer-space allocatable 69 | # This is measured in units of pages (4096 bytes) 70 | "net.ipv4.tcp_mem='65536 131072 262144'", 71 | 72 | "net.ipv4.udp_mem='65536 131072 262144'", 73 | 74 | # Default Socket Receive Buffer 75 | "net.core.rmem_default=25165824", 76 | 77 | # Maximum Socket Receive Buffer 78 | "net.core.rmem_max=25165824 ", 79 | 80 | # Increase the read-buffer space allocatable (minimum size, 81 | # initial size, and maximum size in bytes) 82 | "net.ipv4.tcp_rmem='20480 12582912 25165824'", 83 | 84 | "net.ipv4.udp_rmem_min=16384", 85 | 86 | # Default Socket Send Buffer 87 | "net.core.wmem_default=25165824", 88 | 89 | # Maximum Socket Send Buffer 90 | "net.core.wmem_max=25165824", 91 | 92 | # Increase the write-buffer-space allocatable 93 | "net.ipv4.tcp_wmem='20480 12582912 25165824'", 94 | 95 | "net.ipv4.udp_wmem_min=16384", 96 | 97 | # Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks 98 | "net.ipv4.tcp_max_tw_buckets=1440000", 99 | 100 | "net.ipv4.tcp_tw_reuse=1", 101 | ] 102 | } 103 | -------------------------------------------------------------------------------- /aws/cluster/networking.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * Configures a VPC on top of which we'll use kops to spawn our Kubernetes cluster 3 | */ 4 | 5 | // The VPC itself 6 | resource "aws_vpc" "main" { 7 | cidr_block = var.vpc-cidr 8 | instance_tenancy = "default" 9 | enable_dns_hostnames = true 10 | enable_dns_support = true 11 | 12 | tags = { 13 | Name = var.cluster-name 14 | Origin = "Terraform" 15 | } 16 | 17 | lifecycle { 18 | ignore_changes = ["tags"] 19 | } 20 | } 21 | 22 | // Internet Gateway 23 | resource "aws_internet_gateway" "igw" { 24 | vpc_id = aws_vpc.main.id 25 | 26 | tags = { 27 | Name = var.cluster-name 28 | Origin = "Terraform" 29 | } 30 | } 31 | 32 | resource "aws_subnet" "utility" { 33 | count = length(var.availability-zones) 34 | 35 | vpc_id = aws_vpc.main.id 36 | availability_zone = element(var.availability-zones, count.index) 37 | map_public_ip_on_launch = true 38 | 39 | cidr_block = "${cidrsubnet( 40 | cidrsubnet(aws_vpc.main.cidr_block, 2, 0), 41 | 3, count.index 42 | )}" 43 | 44 | tags = { 45 | Name = "utility-${element(var.availability-zones, count.index)}.${var.cluster-name}" 46 | Origin = "Terraform" 47 | } 48 | 49 | lifecycle { 50 | ignore_changes = ["tags"] 51 | } 52 | } 53 | 54 | resource "aws_route_table" "utility" { 55 | count = length(var.availability-zones) 56 | 57 | vpc_id = aws_vpc.main.id 58 | 59 | tags = { 60 | Name = "utility-${element(var.availability-zones, count.index)}.${var.cluster-name}" 61 | Origin = "Terraform" 62 | } 63 | } 64 | 65 | resource "aws_route" "utility-to-internet" { 66 | count = length(var.availability-zones) 67 | 68 | route_table_id = element(aws_route_table.utility.*.id, count.index) 69 | destination_cidr_block = "0.0.0.0/0" 70 | gateway_id = aws_internet_gateway.igw.id 71 | } 72 | 73 | resource "aws_route_table_association" "utility" { 74 | count = length(var.availability-zones) 75 | 76 | subnet_id = element(aws_subnet.utility.*.id, count.index) 77 | route_table_id = element(aws_route_table.utility.*.id, count.index) 78 | } 79 | 80 | ################################## 81 | ## Private subnets (one per AZ) ## 82 | ################################## 83 | resource "aws_subnet" "private" { 84 | count = length(var.availability-zones) 85 | 86 | vpc_id = aws_vpc.main.id 87 | 88 | cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 3, 2 + count.index) 89 | 90 | availability_zone = element(var.availability-zones, count.index) 91 | 92 | map_public_ip_on_launch = false 93 | 94 | tags = { 95 | Name = "${element(var.availability-zones, count.index)}.${var.cluster-name}" 96 | Origin = "Terraform" 97 | } 98 | 99 | lifecycle { 100 | ignore_changes = ["tags"] 101 | } 102 | } 103 | 104 | resource "aws_eip" "nat-device" { 105 | count = length(var.availability-zones) 106 | 107 | vpc = true 108 | } 109 | 110 | resource "aws_nat_gateway" "natgw" { 111 | count = length(var.availability-zones) 112 | 113 | allocation_id = element(aws_eip.nat-device.*.id, count.index) 114 | subnet_id = element(aws_subnet.utility.*.id, count.index) 115 | } 116 | 117 | resource "aws_route_table" "private" { 118 | count = length(var.availability-zones) 119 | 120 | vpc_id = aws_vpc.main.id 121 | 122 | tags = { 123 | Name = "${element(var.availability-zones, count.index)}.${var.cluster-name}" 124 | Origin = "Terraform" 125 | } 126 | } 127 | 128 | resource "aws_route" "private-to-internet" { 129 | count = length(var.availability-zones) 130 | 131 | route_table_id = element(aws_route_table.private.*.id, count.index) 132 | destination_cidr_block = "0.0.0.0/0" 133 | nat_gateway_id = element(aws_nat_gateway.natgw.*.id, count.index) 134 | } 135 | 136 | resource "aws_route_table_association" "private" { 137 | count = length(var.availability-zones) 138 | 139 | subnet_id = element(aws_subnet.private.*.id, count.index) 140 | route_table_id = element(aws_route_table.private.*.id, count.index) 141 | } 142 | -------------------------------------------------------------------------------- /aws/cluster/main.tf: -------------------------------------------------------------------------------- 1 | // Cluster manifest on S3 and cluster destroy-time provisioner 2 | resource "aws_s3_bucket_object" "cluster-spec" { 3 | bucket = var.kops-state-bucket 4 | key = "/karch-specs/${var.cluster-name}/master-cluster-spec.yml" 5 | 6 | content = < ${path.module}/${var.cluster-name}-cluster-spec.yml 30 | ${aws_s3_bucket_object.cluster-spec.content} 31 | EOF 32 | FILEDUMP 33 | } 34 | 35 | // Let's wait for our newly created DNS zone to propagate 36 | provisioner "local-exec" { 37 | command = < ${path.module}/${var.cluster-name}-cluster-spec.yml 88 | ${aws_s3_bucket_object.cluster-spec.content} 89 | EOF 90 | FILEDUMP 91 | } 92 | 93 | provisioner "local-exec" { 94 | command = < 141 | -------------------------------------------------------------------------------- /aws/test/k8s/kube-system/cluster-autoscaler.yaml: -------------------------------------------------------------------------------- 1 | # Service account and roles 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | k8s-addon: cluster-autoscaler.addons.k8s.io 7 | k8s-app: cluster-autoscaler 8 | name: cluster-autoscaler 9 | namespace: kube-system 10 | --- 11 | apiVersion: rbac.authorization.k8s.io/v1beta1 12 | kind: ClusterRole 13 | metadata: 14 | name: cluster-autoscaler 15 | labels: 16 | k8s-addon: cluster-autoscaler.addons.k8s.io 17 | k8s-app: cluster-autoscaler 18 | rules: 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - events 23 | - endpoints 24 | verbs: 25 | - create 26 | - patch 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - pods/eviction 31 | verbs: 32 | - create 33 | - apiGroups: 34 | - "" 35 | resources: 36 | - pods/status 37 | verbs: 38 | - update 39 | - apiGroups: 40 | - "" 41 | resources: 42 | - endpoints 43 | resourceNames: 44 | - cluster-autoscaler 45 | verbs: 46 | - get 47 | - update 48 | - apiGroups: 49 | - "" 50 | resources: 51 | - nodes 52 | verbs: 53 | - watch 54 | - list 55 | - get 56 | - update 57 | - apiGroups: 58 | - "" 59 | resources: 60 | - pods 61 | - services 62 | - replicationcontrollers 63 | - persistentvolumeclaims 64 | - persistentvolumes 65 | verbs: 66 | - watch 67 | - list 68 | - get 69 | - apiGroups: 70 | - extensions 71 | resources: 72 | - replicasets 73 | - daemonsets 74 | - deployments 75 | verbs: 76 | - watch 77 | - list 78 | - get 79 | - apiGroups: 80 | - policy 81 | resources: 82 | - poddisruptionbudgets 83 | verbs: 84 | - watch 85 | - list 86 | - apiGroups: 87 | - apps 88 | resources: 89 | - statefulsets 90 | verbs: 91 | - watch 92 | - list 93 | - get 94 | 95 | --- 96 | apiVersion: rbac.authorization.k8s.io/v1beta1 97 | kind: ClusterRoleBinding 98 | metadata: 99 | name: cluster-autoscaler 100 | labels: 101 | k8s-addon: cluster-autoscaler.addons.k8s.io 102 | k8s-app: cluster-autoscaler 103 | roleRef: 104 | apiGroup: rbac.authorization.k8s.io 105 | kind: ClusterRole 106 | name: cluster-autoscaler 107 | subjects: 108 | - kind: ServiceAccount 109 | name: cluster-autoscaler 110 | namespace: kube-system 111 | --- 112 | apiVersion: rbac.authorization.k8s.io/v1beta1 113 | kind: Role 114 | metadata: 115 | name: cluster-autoscaler 116 | namespace: kube-system 117 | labels: 118 | k8s-addon: cluster-autoscaler.addons.k8s.io 119 | k8s-app: cluster-autoscaler 120 | rules: 121 | - apiGroups: 122 | - "" 123 | resources: 124 | - configmaps 125 | verbs: 126 | - create 127 | - apiGroups: 128 | - "" 129 | resources: 130 | - configmaps 131 | resourceNames: 132 | - cluster-autoscaler-status 133 | verbs: 134 | - delete 135 | - get 136 | - update 137 | --- 138 | apiVersion: rbac.authorization.k8s.io/v1beta1 139 | kind: RoleBinding 140 | metadata: 141 | name: cluster-autoscaler 142 | namespace: kube-system 143 | labels: 144 | k8s-addon: cluster-autoscaler.addons.k8s.io 145 | k8s-app: cluster-autoscaler 146 | roleRef: 147 | apiGroup: rbac.authorization.k8s.io 148 | kind: Role 149 | name: cluster-autoscaler 150 | subjects: 151 | - kind: ServiceAccount 152 | name: cluster-autoscaler 153 | namespace: kube-system 154 | --- 155 | # PodDisruptionBudget of zero allows the cluster autoscaler to scale down a 156 | # node which has the cluster autoscaler on 157 | apiVersion: policy/v1beta1 158 | kind: PodDisruptionBudget 159 | metadata: 160 | name: cluster-autoscaler 161 | namespace: kube-system 162 | spec: 163 | minAvailable: 0 164 | selector: 165 | matchLabels: 166 | k8s-app: cluster-autoscaler 167 | --- 168 | # Actual cluster autoscaler deployment 169 | apiVersion: extensions/v1beta1 170 | kind: Deployment 171 | metadata: 172 | name: cluster-autoscaler 173 | namespace: kube-system 174 | labels: 175 | k8s-addon: cluster-autoscaler.addons.k8s.io 176 | k8s-app: cluster-autoscaler 177 | spec: 178 | replicas: 1 179 | strategy: 180 | type: RollingUpdate 181 | rollingUpdate: 182 | maxUnavailable: 0 183 | maxSurge: 1 184 | selector: 185 | matchLabels: 186 | k8s-app: cluster-autoscaler 187 | template: 188 | metadata: 189 | labels: 190 | k8s-addon: cluster-autoscaler.addons.k8s.io 191 | k8s-app: cluster-autoscaler 192 | spec: 193 | serviceAccountName: cluster-autoscaler 194 | nodeSelector: 195 | duty: webserver 196 | containers: 197 | - name: cluster-autoscaler 198 | image: gcr.io/google-containers/cluster-autoscaler:v1.0.3 199 | resources: 200 | limits: 201 | cpu: 100m 202 | memory: 300Mi 203 | requests: 204 | cpu: 100m 205 | memory: 300Mi 206 | command: 207 | - ./cluster-autoscaler 208 | - --cloud-provider=aws 209 | - --scale-down-enabled=true 210 | - --scale-down-delay-after-add=1m 211 | - --scale-down-utilization-threshold=0.6 212 | - --scale-down-unneeded-time=1m 213 | - --scale-down-unready-time=6m 214 | - --expander=least-waste 215 | - --stderrthreshold=warning 216 | - --v=4 217 | - --skip-nodes-with-local-storage=false 218 | # Per-ASG configuration 219 | - --nodes=1:5:ingress.test.morpheo.io 220 | - --nodes=1:3:default.test.morpheo.io 221 | env: 222 | - name: AWS_REGION 223 | value: eu-west-1 224 | volumeMounts: 225 | - name: ssl-certs 226 | mountPath: "/etc/ssl/certs/ca-certificates.crt" 227 | readOnly: true 228 | volumes: 229 | - name: ssl-certs 230 | hostPath: 231 | path: "/etc/ssl/certs/ca-certificates.crt" 232 | nodeSelector: 233 | node-role.kubernetes.io/master: "" 234 | tolerations: 235 | - key: "node-role.kubernetes.io/master" 236 | effect: NoSchedule 237 | -------------------------------------------------------------------------------- /aws/cluster/templates/cluster-spec.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kops/v1alpha2 2 | kind: Cluster 3 | metadata: 4 | name: ${cluster-name} 5 | spec: 6 | api: 7 | loadBalancer: 8 | type: ${master-lb-visibility} 9 | idleTimeoutSeconds: ${master-lb-idle-timeout} 10 | authorization: 11 | ${kops-authorization-mode}: {} 12 | channel: ${channel} 13 | cloudConfig: 14 | disableSecurityGroupIngress: ${disable-sg-ingress} 15 | ${elb-security-group} 16 | cloudLabels: 17 | ${cloud-labels} 18 | cloudProvider: aws 19 | clusterDNSDomain: ${kube-dns-domain} 20 | configBase: s3://${kops-state-bucket}/${cluster-name} 21 | configStore: s3://${kops-state-bucket}/${cluster-name} 22 | dnsZone: ${cluster-name} 23 | etcdClusters: 24 | ${etcd-clusters} 25 | keyStore: s3://${kops-state-bucket}/${cluster-name}/pki 26 | kubeAPIServer: 27 | insecureBindAddress: 127.0.0.1 28 | enableAdmissionPlugins: 29 | - NamespaceLifecycle 30 | - LimitRanger 31 | - ServiceAccount 32 | - PersistentVolumeLabel 33 | - DefaultStorageClass 34 | - DefaultTolerationSeconds 35 | - MutatingAdmissionWebhook 36 | - ResourceQuota 37 | - Initializers 38 | - NodeRestriction 39 | - Priority 40 | - PersistentVolumeClaimResize 41 | - PodPreset 42 | allowPrivileged: true 43 | anonymousAuth: false 44 | apiServerCount: ${master-count} 45 | authorizationMode: ${apiserver-authorization-mode} 46 | ${oidc-config} 47 | cloudProvider: aws 48 | etcdServers: 49 | - http://127.0.0.1:4001 50 | etcdServersOverrides: 51 | - /events#http://127.0.0.1:4002 52 | image: gcr.io/google_containers/kube-apiserver:v${kubernetes-version} 53 | insecurePort: 8080 54 | kubeletPreferredAddressTypes: 55 | - InternalIP 56 | - Hostname 57 | - ExternalIP 58 | logLevel: ${log-level} 59 | securePort: 443 60 | serviceClusterIPRange: 100.64.0.0/13 61 | storageBackend: ${apiserver-storage-backend} 62 | runtimeConfig: 63 | ${apiserver-runtime-config} 64 | featureGates: 65 | ExperimentalCriticalPodAnnotation: "true" 66 | ExpandPersistentVolumes: "true" 67 | Initializers: "true" 68 | PodPriority: "true" 69 | kubeControllerManager: 70 | allocateNodeCIDRs: true 71 | attachDetachReconcileSyncPeriod: 1m0s 72 | cloudProvider: aws 73 | clusterCIDR: 100.96.0.0/11 74 | clusterName: ${cluster-name} 75 | configureCloudRoutes: false 76 | image: gcr.io/google_containers/kube-controller-manager:v${kubernetes-version} 77 | leaderElection: 78 | leaderElect: true 79 | logLevel: ${log-level} 80 | useServiceAccountCredentials: true 81 | horizontalPodAutoscalerSyncPerdiod: ${hpa-sync-period} 82 | horizontalPodAutoscalerDownscaleDelay: ${hpa-scale-down-delay} 83 | horizontalPodAutoscalerUpscaleDelay: ${hpa-scale-up-delay} 84 | featureGates: 85 | ExperimentalCriticalPodAnnotation: "true" 86 | ExpandPersistentVolumes: "true" 87 | Initializers: "true" 88 | PodPriority: "true" 89 | 90 | kubeDNS: 91 | provider: ${dns-provider} 92 | domain: ${kube-dns-domain} 93 | serverIP: 100.64.0.10 94 | 95 | kubeProxy: 96 | clusterCIDR: 100.96.0.0/11 97 | cpuRequest: 100m 98 | memoryRequest: 100Mi 99 | hostnameOverride: '@aws' 100 | image: gcr.io/google_containers/kube-proxy:v${kubernetes-version} 101 | logLevel: ${log-level} 102 | proxyMode: ${kube-proxy-mode} 103 | 104 | kubeScheduler: 105 | image: gcr.io/google_containers/kube-scheduler:v${kubernetes-version} 106 | leaderElection: 107 | leaderElect: true 108 | logLevel: ${log-level} 109 | kubelet: 110 | allowPrivileged: true 111 | cgroupRoot: / 112 | cloudProvider: aws 113 | clusterDNS: 100.64.0.10 114 | clusterDomain: ${kube-dns-domain} 115 | enableDebuggingHandlers: true 116 | evictionHard: ${kubelet-eviction-flag} 117 | hostnameOverride: '@aws' 118 | kubeconfigPath: /var/lib/kubelet/kubeconfig 119 | logLevel: ${log-level} 120 | networkPluginName: cni 121 | nonMasqueradeCIDR: 100.64.0.0/10 122 | podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0 123 | podManifestPath: /etc/kubernetes/manifests 124 | kubeReserved: 125 | cpu: "${kube-reserved-cpu}" 126 | memory: "${kube-reserved-memory}" 127 | systemReserved: 128 | cpu: "${system-reserved-cpu}" 129 | memory: "${system-reserved-memory}" 130 | enforceNodeAllocatable: "pods" 131 | featureGates: 132 | ExperimentalCriticalPodAnnotation: "true" 133 | ExpandPersistentVolumes: "true" 134 | Initializers: "true" 135 | PodPriority: "true" 136 | kubernetesApiAccess: 137 | ${trusted-cidrs} 138 | kubernetesVersion: ${kubernetes-version} 139 | masterInternalName: api.internal.${cluster-name} 140 | masterKubelet: 141 | allowPrivileged: true 142 | cgroupRoot: / 143 | cloudProvider: aws 144 | clusterDNS: 100.64.0.10 145 | clusterDomain: ${kube-dns-domain} 146 | enableDebuggingHandlers: true 147 | evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% 148 | hostnameOverride: '@aws' 149 | kubeconfigPath: /var/lib/kubelet/kubeconfig 150 | logLevel: ${log-level} 151 | networkPluginName: cni 152 | nonMasqueradeCIDR: 100.64.0.0/10 153 | podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0 154 | podManifestPath: /etc/kubernetes/manifests 155 | registerSchedulable: false 156 | masterPublicName: api.${cluster-name} 157 | networkCIDR: ${vpc-cidr} 158 | networkID: ${vpc-id} 159 | networking: 160 | ${container-networking}: 161 | ${networking-config} 162 | nonMasqueradeCIDR: 100.64.0.0/10 163 | secretStore: s3://${kops-state-bucket}/${cluster-name}/secrets 164 | serviceClusterIPRange: 100.64.0.0/13 165 | sshAccess: 166 | ${trusted-cidrs} 167 | subnets: 168 | ${subnets} 169 | topology: 170 | bastion: 171 | bastionPublicName: bastion.${cluster-name} 172 | dns: 173 | type: ${master-lb-dns-visibility} 174 | masters: private 175 | nodes: private 176 | hooks: 177 | ${hooks} 178 | additionalPolicies: 179 | ${master-additional-policies} 180 | ${node-additional-policies} 181 | -------------------------------------------------------------------------------- /aws/cluster/cluster-spec-template.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "cluster-spec" { 2 | template = file("${path.module}/templates/cluster-spec.yaml") 3 | 4 | vars = { 5 | # Generic cluster configuration 6 | cluster-name = var.cluster-name 7 | kubernetes-version = var.kubernetes-version 8 | channel = var.channel 9 | cloud-labels = join("\n", data.template_file.cloud-labels.*.rendered) 10 | kube-dns-domain = var.kube-dns-domain 11 | kops-state-bucket = var.kops-state-bucket 12 | 13 | # Control plane HA mode and network exposure configuration 14 | master-lb-visibility = var.master-lb-visibility == "Private" ? "Internal" : "Public" 15 | master-lb-dns-visibility = var.master-lb-visibility 16 | master-count = length(var.master-availability-zones) 17 | master-lb-idle-timeout = var.master-lb-idle-timeout 18 | 19 | # Cloud provider networking configuration 20 | vpc-cidr = aws_vpc.main.cidr_block 21 | vpc-id = aws_vpc.main.id 22 | trusted-cidrs = join("\n", data.template_file.trusted-cidrs.*.rendered) 23 | subnets = join("\n", data.template_file.subnets.*.rendered) 24 | 25 | # DNS provider to use 26 | dns-provider = var.dns-provider 27 | 28 | # Kube proxy mode 29 | kube-proxy-mode = var.kube-proxy-mode 30 | 31 | # CNI plugin to use 32 | container-networking = var.container-networking 33 | networking-config = data.template_file.networking-config.rendered 34 | 35 | # Extra systemd hooks for all nodes in our cluster 36 | hooks = join("\n", data.template_file.hooks.*.rendered) 37 | 38 | # ETCD cluster parameters 39 | etcd-clusters = < 0 ? "additionalSecurityGroups:" : ""} 14 | ${join("\n", data.template_file.master-additional-sgs.*.rendered)} 15 | EOF 16 | 17 | image = var.master-image 18 | type = var.master-machine-type 19 | max-size = 1 20 | min-size = 1 21 | role = "Master" 22 | volume-size = var.master-volume-size 23 | volume-provisioned-iops = var.master-volume-provisioned-iops == "" ? "" : var.master-volume-provisioned-iops 24 | volume-type = var.master-volume-type 25 | ebs-optimized = var.master-ebs-optimized 26 | max-price = "" 27 | taints = "" 28 | subnets = " - ${element(var.master-availability-zones, count.index)}" 29 | hooks = join("\n", data.template_file.master-hooks.*.rendered) 30 | } 31 | } 32 | 33 | data "template_file" "master-additional-sgs" { 34 | count = var.master-additional-sgs-count 35 | 36 | template = " - $${sg-id}" 37 | 38 | vars = { 39 | sg-id = element(var.master-additional-sgs, count.index) 40 | } 41 | } 42 | 43 | data "template_file" "master-cloud-labels" { 44 | count = length(keys(var.master-cloud-labels)) 45 | 46 | template = < 0 ? "additionalSecurityGroups:" : ""} 90 | ${join("\n", data.template_file.bastion-additional-sgs.*.rendered)} 91 | EOF 92 | 93 | image = var.bastion-image 94 | type = var.bastion-machine-type 95 | max-size = var.max-bastions 96 | min-size = var.min-bastions 97 | role = "Bastion" 98 | volume-size = var.bastion-volume-size 99 | volume-provisioned-iops = var.bastion-volume-provisioned-iops == "" ? "" : var.bastion-volume-provisioned-iops 100 | volume-type = var.bastion-volume-type 101 | ebs-optimized = var.bastion-ebs-optimized 102 | max-price = "" 103 | taints = "" 104 | subnets = join("\n", data.template_file.minion-subnets.*.rendered) 105 | hooks = join("\n", data.template_file.bastion-hooks.*.rendered) 106 | } 107 | } 108 | 109 | data "template_file" "bastion-additional-sgs" { 110 | count = var.bastion-additional-sgs-count 111 | 112 | template = " - $${sg-id}" 113 | 114 | vars = { 115 | sg-id = element(var.bastion-additional-sgs, count.index) 116 | } 117 | } 118 | 119 | data "template_file" "bastion-cloud-labels" { 120 | count = length(keys(var.bastion-cloud-labels)) 121 | 122 | template = < 0 ? "additionalSecurityGroups:" : ""} 165 | ${join("\n", data.template_file.minion-additional-sgs.*.rendered)} 166 | EOF 167 | 168 | image = var.minion-image 169 | type = var.minion-machine-type 170 | max-size = var.max-minions 171 | min-size = var.min-minions 172 | role = "Node" 173 | volume-size = var.minion-volume-size 174 | volume-provisioned-iops = var.minion-volume-provisioned-iops == "" ? "" : var.minion-volume-provisioned-iops 175 | volume-type = var.minion-volume-type 176 | ebs-optimized = var.minion-ebs-optimized 177 | max-price = "" 178 | taints = join("\n", data.template_file.minion-taints.*.rendered) 179 | subnets = join("\n", data.template_file.minion-subnets.*.rendered) 180 | hooks = join("\n", data.template_file.minion-hooks.*.rendered) 181 | } 182 | } 183 | 184 | data "template_file" "minion-taints" { 185 | count = length(var.minion-taints) 186 | 187 | template = " - ${element(var.minion-taints, count.index)}" 188 | } 189 | 190 | data "template_file" "minion-subnets" { 191 | count = length(var.availability-zones) 192 | template = " - $${az}" 193 | 194 | vars = { 195 | az = element(var.availability-zones, count.index) 196 | } 197 | } 198 | 199 | data "template_file" "minion-additional-sgs" { 200 | count = var.minion-additional-sgs-count 201 | 202 | template = " - $${sg-id}" 203 | 204 | vars = { 205 | sg-id = element(var.minion-additional-sgs, count.index) 206 | } 207 | } 208 | 209 | data "template_file" "minion-cloud-labels" { 210 | count = length(keys(var.minion-cloud-labels)) 211 | 212 | template = <-" 96 | # Here: "-" 97 | # This has to be adapted if you change either parameter 98 | # when launching the nginx-ingress-controller. 99 | - "ingress-controller-leader-nginx" 100 | verbs: 101 | - get 102 | - update 103 | - apiGroups: 104 | - "" 105 | resources: 106 | - configmaps 107 | verbs: 108 | - create 109 | - apiGroups: 110 | - "" 111 | resources: 112 | - endpoints 113 | verbs: 114 | - get 115 | - create 116 | - update 117 | --- 118 | apiVersion: rbac.authorization.k8s.io/v1beta1 119 | kind: RoleBinding 120 | metadata: 121 | labels: 122 | k8s-addon: ingress-nginx.addons.k8s.io 123 | name: nginx-ingress-controller 124 | namespace: ingress 125 | roleRef: 126 | apiGroup: rbac.authorization.k8s.io 127 | kind: Role 128 | name: nginx-ingress-controller 129 | subjects: 130 | - kind: ServiceAccount 131 | name: nginx-ingress-controller 132 | namespace: ingress 133 | --- 134 | # Ingress Controller config 135 | kind: ConfigMap 136 | apiVersion: v1 137 | metadata: 138 | name: nginx-ingress-controller 139 | namespace: ingress 140 | data: 141 | # Feature gates 142 | enable-vts-status: "true" 143 | enable-brotli: "false" 144 | use-gzip: "false" 145 | use-http2: "false" 146 | use-proxy-protocol: "true" 147 | 148 | # Proxy parameters 149 | worker-processes: "auto" 150 | max-worker-connections: "10000" 151 | 152 | proxy-body-size: "0" 153 | proxy-connect-timeout: "15" 154 | proxy-read-timeout: "600" 155 | proxy-send-timeout: "60" 156 | 157 | ## Retry behaviour (can be changed depending on how you handle the performance 158 | ## vs robustness tradeoff: if proxy request buffering is enabled, client 159 | ## requests will be written to RAM/disk before they are sent to the client 160 | ## which makes it possible to retry the request against a second backend in 161 | ## case a timeout/5xx error occurs... but this obviously requires extra 162 | ## RAM/disk space) 163 | proxy-request-buffering: "off" 164 | upstream-fail-timeout: "5" 165 | upstream-max-fails: "3" 166 | 167 | # TLS parameters 168 | ssl-protocols: "TLSv1 TLSv1.1 TLSv1.2" 169 | ssl-redirect: "true" 170 | hsts: "true" 171 | hsts-max-age: "100000" 172 | hsts-include-subdomains: "false" 173 | ssl-ciphers: "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" 174 | 175 | log-format-upstream: '{ "time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x-forward-for": "$proxy_add_x_forwarded_for", "request_id": "$request_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent" }' 176 | --- 177 | apiVersion: policy/v1beta1 178 | kind: PodDisruptionBudget 179 | metadata: 180 | name: ingress-nginx 181 | namespace: ingress 182 | spec: 183 | minAvailable: 1 184 | selector: 185 | matchLabels: 186 | k8s-app: nginx-ingress-controller 187 | --- 188 | apiVersion: autoscaling/v1 189 | kind: HorizontalPodAutoscaler 190 | metadata: 191 | name: ingress-nginx 192 | namespace: ingress 193 | spec: 194 | scaleTargetRef: 195 | apiVersion: extensions/v1beta1 196 | kind: Deployment 197 | name: ingress-nginx 198 | minReplicas: 1 199 | maxReplicas: 5 200 | targetCPUUtilizationPercentage: 30 201 | --- 202 | kind: Service 203 | apiVersion: v1 204 | metadata: 205 | name: ingress-nginx 206 | namespace: ingress 207 | labels: 208 | k8s-app: nginx-ingress-controller 209 | annotations: 210 | # Enable PROXY protocol 211 | service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: '*' 212 | # Increase the ELB idle timeout to avoid issues with WebSockets or Server-Sent Events. 213 | service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' 214 | # Enable cross-zone load-balancing 215 | service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true' 216 | spec: 217 | type: LoadBalancer 218 | selector: 219 | k8s-app: nginx-ingress-controller 220 | ports: 221 | - name: http 222 | port: 80 223 | targetPort: http 224 | - name: https 225 | port: 443 226 | targetPort: https 227 | --- 228 | # Ingress controller deployment 229 | kind: Deployment 230 | apiVersion: extensions/v1beta1 231 | metadata: 232 | name: ingress-nginx 233 | namespace: ingress 234 | labels: 235 | k8s-app: nginx-ingress-controller 236 | k8s-addon: ingress-nginx.addons.k8s.io 237 | karch-test: sure 238 | spec: 239 | strategy: 240 | type: RollingUpdate 241 | rollingUpdate: 242 | maxUnavailable: 1 243 | maxSurge: 0 244 | template: 245 | metadata: 246 | labels: 247 | app: ingress-nginx 248 | k8s-app: nginx-ingress-controller 249 | k8s-addon: ingress-nginx.addons.k8s.io 250 | annotations: 251 | prometheus.io/port: '10254' 252 | prometheus.io/scrape: 'true' 253 | spec: 254 | nodeSelector: 255 | duty: intake 256 | affinity: 257 | podAntiAffinity: 258 | requiredDuringSchedulingIgnoredDuringExecution: 259 | - labelSelector: 260 | matchExpressions: 261 | - key: k8s-app 262 | operator: In 263 | values: 264 | - nginx-ingress-controller 265 | topologyKey: kubernetes.io/hostname 266 | terminationGracePeriodSeconds: 10 267 | serviceAccountName: nginx-ingress-controller 268 | volumes: 269 | - name: nginx-conf-template 270 | configMap: 271 | name: nginx-conf-template 272 | containers: 273 | - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 274 | name: nginx-ingress-controller 275 | imagePullPolicy: Always 276 | volumeMounts: 277 | - name: nginx-conf-template 278 | mountPath: /etc/nginx/template 279 | readOnly: true 280 | ports: 281 | - name: http 282 | containerPort: 80 283 | protocol: TCP 284 | - name: https 285 | containerPort: 443 286 | protocol: TCP 287 | - name: status 288 | containerPort: 18080 289 | protocol: TCP 290 | readinessProbe: 291 | httpGet: 292 | path: /healthz 293 | port: 10254 294 | scheme: HTTP 295 | initialDelaySeconds: 1 296 | timeoutSeconds: 5 297 | livenessProbe: 298 | httpGet: 299 | path: /healthz 300 | port: 10254 301 | scheme: HTTP 302 | initialDelaySeconds: 10 303 | timeoutSeconds: 5 304 | resources: 305 | limits: 306 | memory: 400Mi 307 | requests: 308 | cpu: 500m 309 | memory: 400Mi 310 | env: 311 | - name: POD_NAME 312 | valueFrom: 313 | fieldRef: 314 | fieldPath: metadata.name 315 | - name: POD_NAMESPACE 316 | valueFrom: 317 | fieldRef: 318 | fieldPath: metadata.namespace 319 | args: 320 | - /nginx-ingress-controller 321 | - --default-backend-service=$(POD_NAMESPACE)/default-backend 322 | - --configmap=$(POD_NAMESPACE)/nginx-ingress-controller 323 | - --publish-service=$(POD_NAMESPACE)/ingress-nginx 324 | - --sort-backends 325 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /aws/cluster/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws-region" { 2 | description = "The AWS region in which to deploy your cluster & VPC." 3 | type = "string" 4 | } 5 | 6 | # Networking & Security 7 | variable "vpc-name" { 8 | description = "Arbitrary name to give to your VPC" 9 | type = "string" 10 | } 11 | 12 | variable "vpc-cidr" { 13 | description = "CIDR of the VPC housing the Kubernetes cluster" 14 | type = "string" 15 | } 16 | 17 | variable "availability-zones" { 18 | type = "list" 19 | description = "Availability zones to span (for HA master deployments, see master-availability-zones)" 20 | } 21 | 22 | variable "kops-topology" { 23 | type = "string" 24 | description = "Kops topolopy (public|private), (default: private)" 25 | default = "private" 26 | } 27 | 28 | variable "trusted-cidrs" { 29 | type = "list" 30 | description = "CIDR whitelist for Kubernetes master HTTPs & bastion SSH access (default: 0.0.0.0/0)" 31 | 32 | default = ["0.0.0.0/0"] 33 | } 34 | 35 | variable "admin-ssh-public-key-path" { 36 | type = "string" 37 | description = "Path to the cluster admin's public SSH key (default: ~/.ssh/id_rsa.pub)" 38 | 39 | default = "~/.ssh/id_rsa.pub" 40 | } 41 | 42 | variable "extra-route53-vpc-associations" { 43 | type = "list" 44 | description = < 16 thousand 64-byte states or about 8 thousand 128-byte states */}} 373 | {{ range $zone := (buildRateLimitZones $servers) }} 374 | {{ $zone }} 375 | {{ end }} 376 | 377 | {{/* Build server redirects (from/to www) */}} 378 | {{ range $hostname, $to := .RedirectServers }} 379 | server { 380 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 381 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 382 | listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl; 383 | {{ else }} 384 | listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 385 | listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl; 386 | {{ end }} 387 | {{ if $IsIPV6Enabled }} 388 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 389 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 390 | listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}; 391 | {{ else }} 392 | listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; 393 | listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}; 394 | {{ end }} 395 | {{ end }} 396 | server_name {{ $hostname }}; 397 | 398 | {{ if ne $all.ListenPorts.HTTPS 443 }} 399 | {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} 400 | return {{ $all.Cfg.HTTPRedirectCode }} $scheme://{{ $to }}{{ $redirect_port }}$request_uri; 401 | {{ else }} 402 | return {{ $all.Cfg.HTTPRedirectCode }} $scheme://{{ $to }}$request_uri; 403 | {{ end }} 404 | } 405 | {{ end }} 406 | 407 | {{ range $index, $server := $servers }} 408 | 409 | ## start server {{ $server.Hostname }} 410 | server { 411 | server_name {{ $server.Hostname }} {{ $server.Alias }}; 412 | {{ template "SERVER" serverConfig $all $server }} 413 | 414 | {{ if not (empty $cfg.ServerSnippet) }} 415 | # Custom code snippet configured in the configuration configmap 416 | {{ $cfg.ServerSnippet }} 417 | {{ end }} 418 | 419 | {{ template "CUSTOM_ERRORS" $all }} 420 | } 421 | ## end server {{ $server.Hostname }} 422 | 423 | {{ end }} 424 | 425 | # default server, used for NGINX healthcheck and access to nginx stats 426 | server { 427 | # Use the port {{ $all.ListenPorts.Status }} (random value just to avoid known ports) as default port for nginx. 428 | # Changing this value requires a change in: 429 | # https://github.com/kubernetes/ingress-nginx/blob/master/controllers/nginx/pkg/cmd/controller/nginx.go 430 | listen {{ $all.ListenPorts.Status }} default_server reuseport backlog={{ $all.BacklogSize }}; 431 | {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Status }} default_server reuseport backlog={{ $all.BacklogSize }};{{ end }} 432 | set $proxy_upstream_name "-"; 433 | 434 | location {{ $healthzURI }} { 435 | access_log off; 436 | return 200; 437 | } 438 | 439 | location /nginx_status { 440 | set $proxy_upstream_name "internal"; 441 | 442 | {{ if $cfg.EnableVtsStatus }} 443 | vhost_traffic_status_display; 444 | vhost_traffic_status_display_format html; 445 | {{ else }} 446 | access_log off; 447 | stub_status on; 448 | {{ end }} 449 | } 450 | 451 | location / { 452 | {{ if .CustomErrors }} 453 | proxy_set_header X-Code 404; 454 | {{ end }} 455 | set $proxy_upstream_name "upstream-default-backend"; 456 | proxy_pass http://upstream-default-backend; 457 | } 458 | 459 | {{ template "CUSTOM_ERRORS" $all }} 460 | } 461 | } 462 | 463 | stream { 464 | log_format log_stream {{ $cfg.LogFormatStream }}; 465 | 466 | {{ if $cfg.DisableAccessLog }} 467 | access_log off; 468 | {{ else }} 469 | access_log {{ $cfg.AccessLogPath }} log_stream; 470 | {{ end }} 471 | 472 | error_log {{ $cfg.ErrorLogPath }}; 473 | 474 | # TCP services 475 | {{ range $i, $tcpServer := .TCPBackends }} 476 | upstream tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }} { 477 | {{ range $j, $endpoint := $tcpServer.Endpoints }} 478 | server {{ $endpoint.Address }}:{{ $endpoint.Port }}; 479 | {{ end }} 480 | } 481 | server { 482 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 483 | listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 484 | {{ else }} 485 | listen {{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 486 | {{ end }} 487 | {{ if $IsIPV6Enabled }} 488 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 489 | listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 490 | {{ else }} 491 | listen [::]:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; 492 | {{ end }} 493 | {{ end }} 494 | proxy_timeout {{ $cfg.ProxyStreamTimeout }}; 495 | proxy_pass tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}; 496 | {{ if $tcpServer.Backend.ProxyProtocol.Encode }} 497 | proxy_protocol on; 498 | {{ end }} 499 | } 500 | 501 | {{ end }} 502 | 503 | # UDP services 504 | {{ range $i, $udpServer := .UDPBackends }} 505 | upstream udp-{{ $udpServer.Port }}-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }} { 506 | {{ range $j, $endpoint := $udpServer.Endpoints }} 507 | server {{ $endpoint.Address }}:{{ $endpoint.Port }}; 508 | {{ end }} 509 | } 510 | 511 | server { 512 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 513 | listen {{ $address }}:{{ $udpServer.Port }} udp; 514 | {{ else }} 515 | listen {{ $udpServer.Port }} udp; 516 | {{ end }} 517 | {{ if $IsIPV6Enabled }} 518 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 519 | listen {{ $address }}:{{ $udpServer.Port }} udp; 520 | {{ else }} 521 | listen [::]:{{ $udpServer.Port }} udp; 522 | {{ end }} 523 | {{ end }} 524 | proxy_responses {{ $cfg.ProxyStreamResponses }}; 525 | proxy_timeout {{ $cfg.ProxyStreamTimeout }}; 526 | proxy_pass udp-{{ $udpServer.Port }}-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }}; 527 | } 528 | 529 | {{ end }} 530 | } 531 | 532 | {{/* definition of templates to avoid repetitions */}} 533 | {{ define "CUSTOM_ERRORS" }} 534 | {{ $proxySetHeaders := .ProxySetHeaders }} 535 | {{ range $errCode := .Cfg.CustomHTTPErrors }} 536 | location @custom_{{ $errCode }} { 537 | internal; 538 | 539 | proxy_intercept_errors off; 540 | 541 | proxy_set_header X-Code {{ $errCode }}; 542 | proxy_set_header X-Format $http_accept; 543 | proxy_set_header X-Original-URI $request_uri; 544 | proxy_set_header X-Namespace $namespace; 545 | proxy_set_header X-Ingress-Name $ingress_name; 546 | proxy_set_header X-Service-Name $service_name; 547 | 548 | rewrite (.*) / break; 549 | proxy_pass http://upstream-default-backend; 550 | } 551 | {{ end }} 552 | {{ end }} 553 | 554 | {{/* CORS support from https://michielkalkman.com/snippets/nginx-cors-open-configuration.html */}} 555 | {{ define "CORS" }} 556 | {{ $cors := .CorsConfig }} 557 | # Cors Preflight methods needs additional options and different Return Code 558 | if ($request_method = 'OPTIONS') { 559 | add_header 'Access-Control-Allow-Origin' '{{ $cors.CorsAllowOrigin }}' always; 560 | {{ if $cors.CorsAllowCredentials }} add_header 'Access-Control-Allow-Credentials' '{{ $cors.CorsAllowCredentials }}' always; {{ end }} 561 | add_header 'Access-Control-Allow-Methods' '{{ $cors.CorsAllowMethods }}' always; 562 | add_header 'Access-Control-Allow-Headers' '{{ $cors.CorsAllowHeaders }}' always; 563 | add_header 'Access-Control-Max-Age' 1728000; 564 | add_header 'Content-Type' 'text/plain charset=UTF-8'; 565 | add_header 'Content-Length' 0; 566 | return 204; 567 | } 568 | 569 | add_header 'Access-Control-Allow-Origin' '{{ $cors.CorsAllowOrigin }}' always; 570 | {{ if $cors.CorsAllowCredentials }} add_header 'Access-Control-Allow-Credentials' '{{ $cors.CorsAllowCredentials }}' always; {{ end }} 571 | add_header 'Access-Control-Allow-Methods' '{{ $cors.CorsAllowMethods }}' always; 572 | add_header 'Access-Control-Allow-Headers' '{{ $cors.CorsAllowHeaders }}' always; 573 | 574 | {{ end }} 575 | 576 | {{/* definition of server-template to avoid repetitions with server-alias */}} 577 | {{ define "SERVER" }} 578 | {{ $all := .First }} 579 | {{ $server := .Second }} 580 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 581 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}}; 582 | {{ else }} 583 | listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}}; 584 | {{ end }} 585 | {{ if $all.IsIPV6Enabled }} 586 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 587 | listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; 588 | {{ else }} 589 | listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; 590 | {{ end }} 591 | {{ end }} 592 | set $proxy_upstream_name "-"; 593 | 594 | {{/* Listen on {{ $all.ListenPorts.SSLProxy }} because port {{ $all.ListenPorts.HTTPS }} is used in the TLS sni server */}} 595 | {{/* This listener must always have proxy_protocol enabled, because the SNI listener forwards on source IP info in it. */}} 596 | {{ if not (empty $server.SSLCertificate) }} 597 | {{ range $address := $all.Cfg.BindAddressIpv4 }} 598 | listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 599 | {{ else }} 600 | listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 601 | {{ end }} 602 | {{ if $all.IsIPV6Enabled }} 603 | {{ range $address := $all.Cfg.BindAddressIpv6 }} 604 | {{ if not (empty $server.SSLCertificate) }}listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 605 | {{ else }} 606 | {{ if not (empty $server.SSLCertificate) }}listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; 607 | {{ end }} 608 | {{ end }} 609 | {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} 610 | # PEM sha: {{ $server.SSLPemChecksum }} 611 | ssl_certificate {{ $server.SSLCertificate }}; 612 | ssl_certificate_key {{ $server.SSLCertificate }}; 613 | {{ if not (empty $server.SSLFullChainCertificate)}} 614 | ssl_trusted_certificate {{ $server.SSLFullChainCertificate }}; 615 | ssl_stapling on; 616 | ssl_stapling_verify on; 617 | {{ end }} 618 | {{ end }} 619 | 620 | {{ if (and (not (empty $server.SSLCertificate)) $all.Cfg.HSTS) }} 621 | more_set_headers "Strict-Transport-Security: max-age={{ $all.Cfg.HSTSMaxAge }}{{ if $all.Cfg.HSTSIncludeSubdomains }}; includeSubDomains{{ end }};{{ if $all.Cfg.HSTSPreload }} preload{{ end }}"; 622 | {{ end }} 623 | 624 | 625 | {{ if not (empty $server.CertificateAuth.CAFileName) }} 626 | # PEM sha: {{ $server.CertificateAuth.PemSHA }} 627 | ssl_client_certificate {{ $server.CertificateAuth.CAFileName }}; 628 | ssl_verify_client {{ $server.CertificateAuth.VerifyClient }}; 629 | ssl_verify_depth {{ $server.CertificateAuth.ValidationDepth }}; 630 | {{ if not (empty $server.CertificateAuth.ErrorPage)}} 631 | error_page 495 496 = {{ $server.CertificateAuth.ErrorPage }}; 632 | {{ end }} 633 | {{ end }} 634 | 635 | {{ if not (empty $server.ServerSnippet) }} 636 | {{ $server.ServerSnippet }} 637 | {{ end }} 638 | 639 | {{ range $location := $server.Locations }} 640 | {{ $path := buildLocation $location }} 641 | {{ $authPath := buildAuthLocation $location }} 642 | 643 | {{ if not (empty $location.Rewrite.AppRoot)}} 644 | if ($uri = /) { 645 | return 302 {{ $location.Rewrite.AppRoot }}; 646 | } 647 | {{ end }} 648 | 649 | {{ if not (empty $authPath) }} 650 | location = {{ $authPath }} { 651 | internal; 652 | set $proxy_upstream_name "external-authentication"; 653 | 654 | proxy_pass_request_body off; 655 | proxy_set_header Content-Length ""; 656 | 657 | {{ if not (empty $location.ExternalAuth.Method) }} 658 | proxy_method {{ $location.ExternalAuth.Method }}; 659 | proxy_set_header X-Original-URI $request_uri; 660 | proxy_set_header X-Scheme $pass_access_scheme; 661 | {{ end }} 662 | 663 | proxy_set_header Host {{ $location.ExternalAuth.Host }}; 664 | proxy_set_header X-Original-URL $scheme://$http_host$request_uri; 665 | proxy_set_header X-Original-Method $request_method; 666 | proxy_set_header X-Auth-Request-Redirect $request_uri; 667 | proxy_set_header X-Sent-From "nginx-ingress-controller"; 668 | 669 | proxy_http_version 1.1; 670 | proxy_ssl_server_name on; 671 | proxy_pass_request_headers on; 672 | client_max_body_size "{{ $location.Proxy.BodySize }}"; 673 | {{ if isValidClientBodyBufferSize $location.ClientBodyBufferSize }} 674 | client_body_buffer_size {{ $location.ClientBodyBufferSize }}; 675 | {{ end }} 676 | 677 | set $target {{ $location.ExternalAuth.URL }}; 678 | proxy_pass $target; 679 | } 680 | {{ end }} 681 | 682 | location {{ $path }} { 683 | {{ if $all.Cfg.EnableVtsStatus }}{{ if $location.VtsFilterKey }} vhost_traffic_status_filter_by_set_key {{ $location.VtsFilterKey }};{{ end }}{{ end }} 684 | 685 | set $proxy_upstream_name "{{ buildUpstreamName $server.Hostname $all.Backends $location }}"; 686 | 687 | {{ $ing := (getIngressInformation $location.Ingress $path) }} 688 | {{/* $ing.Metadata contains the Ingress metadata */}} 689 | set $namespace "{{ $ing.Namespace }}"; 690 | set $ingress_name "{{ $ing.Rule }}"; 691 | set $service_name "{{ $ing.Service }}"; 692 | 693 | {{ if (or $location.Rewrite.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Rewrite.SSLRedirect)) }} 694 | # enforce ssl on server side 695 | if ($pass_access_scheme = http) { 696 | {{ if ne $all.ListenPorts.HTTPS 443 }} 697 | {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} 698 | return {{ $all.Cfg.HTTPRedirectCode }} https://$best_http_host{{ $redirect_port }}$request_uri; 699 | {{ else }} 700 | return {{ $all.Cfg.HTTPRedirectCode }} https://$best_http_host$request_uri; 701 | {{ end }} 702 | } 703 | {{ end }} 704 | 705 | {{ if $all.Cfg.EnableModsecurity }} 706 | modsecurity on; 707 | 708 | modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; 709 | {{ if $all.Cfg.EnableOWASPCoreRules }} 710 | modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; 711 | {{ end }} 712 | {{ end }} 713 | 714 | {{ if isLocationAllowed $location }} 715 | {{ if gt (len $location.Whitelist.CIDR) 0 }} 716 | if ({{ buildDenyVariable (print $server.Hostname "_" $path) }}) { 717 | return 403; 718 | } 719 | {{ end }} 720 | 721 | port_in_redirect {{ if $location.UsePortInRedirects }}on{{ else }}off{{ end }}; 722 | 723 | {{ if not (empty $authPath) }} 724 | # this location requires authentication 725 | auth_request {{ $authPath }}; 726 | auth_request_set $auth_cookie $upstream_http_set_cookie; 727 | add_header Set-Cookie $auth_cookie; 728 | {{- range $idx, $line := buildAuthResponseHeaders $location }} 729 | {{ $line }} 730 | {{- end }} 731 | {{ end }} 732 | 733 | {{ if not (empty $location.ExternalAuth.SigninURL) }} 734 | error_page 401 = {{ buildAuthSignURL $location.ExternalAuth.SigninURL }}; 735 | {{ end }} 736 | 737 | {{/* if the location contains a rate limit annotation, create one */}} 738 | {{ $limits := buildRateLimit $location }} 739 | {{ range $limit := $limits }} 740 | {{ $limit }}{{ end }} 741 | 742 | {{ if $location.BasicDigestAuth.Secured }} 743 | {{ if eq $location.BasicDigestAuth.Type "basic" }} 744 | auth_basic "{{ $location.BasicDigestAuth.Realm }}"; 745 | auth_basic_user_file {{ $location.BasicDigestAuth.File }}; 746 | {{ else }} 747 | auth_digest "{{ $location.BasicDigestAuth.Realm }}"; 748 | auth_digest_user_file {{ $location.BasicDigestAuth.File }}; 749 | {{ end }} 750 | proxy_set_header Authorization ""; 751 | {{ end }} 752 | 753 | {{ if $location.CorsConfig.CorsEnabled }} 754 | {{ template "CORS" $location }} 755 | {{ end }} 756 | 757 | {{ if not (empty $location.Redirect.URL) }} 758 | if ($uri ~* {{ $path }}) { 759 | return {{ $location.Redirect.Code }} {{ $location.Redirect.URL }}; 760 | } 761 | {{ end }} 762 | 763 | client_max_body_size "{{ $location.Proxy.BodySize }}"; 764 | {{ if isValidClientBodyBufferSize $location.ClientBodyBufferSize }} 765 | client_body_buffer_size {{ $location.ClientBodyBufferSize }}; 766 | {{ end }} 767 | 768 | {{/* By default use vhost as Host to upstream, but allow overrides */}} 769 | {{ if not (empty $location.UpstreamVhost) }} 770 | proxy_set_header Host "{{ $location.UpstreamVhost }}"; 771 | {{ else }} 772 | proxy_set_header Host $best_http_host; 773 | {{ end }} 774 | 775 | 776 | # Pass the extracted client certificate to the backend 777 | {{ if not (empty $server.CertificateAuth.CAFileName) }} 778 | {{ if $server.CertificateAuth.PassCertToUpstream }} 779 | proxy_set_header ssl-client-cert $ssl_client_escaped_cert; 780 | {{ else }} 781 | proxy_set_header ssl-client-cert ""; 782 | {{ end }} 783 | proxy_set_header ssl-client-verify $ssl_client_verify; 784 | proxy_set_header ssl-client-dn $ssl_client_s_dn; 785 | {{ else }} 786 | proxy_set_header ssl-client-cert ""; 787 | proxy_set_header ssl-client-verify ""; 788 | proxy_set_header ssl-client-dn ""; 789 | {{ end }} 790 | 791 | # Allow websocket connections 792 | proxy_set_header Upgrade $http_upgrade; 793 | proxy_set_header Connection $connection_upgrade; 794 | 795 | proxy_set_header X-Real-IP $the_real_ip; 796 | {{ if $all.Cfg.ComputeFullForwardedFor }} 797 | proxy_set_header X-Forwarded-For $full_x_forwarded_for; 798 | {{ else }} 799 | proxy_set_header X-Forwarded-For $the_real_ip; 800 | {{ end }} 801 | proxy_set_header X-Forwarded-Host $best_http_host; 802 | proxy_set_header X-Forwarded-Port $pass_port; 803 | proxy_set_header X-Forwarded-Proto $pass_access_scheme; 804 | proxy_set_header X-Original-URI $request_uri; 805 | proxy_set_header X-Scheme $pass_access_scheme; 806 | 807 | # Pass the original X-Forwarded-For 808 | proxy_set_header X-Original-Forwarded-For {{ buildForwardedFor $all.Cfg.ForwardedForHeader }}; 809 | 810 | # mitigate HTTPoxy Vulnerability 811 | # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ 812 | proxy_set_header Proxy ""; 813 | 814 | # Custom headers to proxied server 815 | {{ range $k, $v := $all.ProxySetHeaders }} 816 | proxy_set_header {{ $k }} "{{ $v }}"; 817 | {{ end }} 818 | 819 | proxy_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; 820 | proxy_send_timeout {{ $location.Proxy.SendTimeout }}s; 821 | proxy_read_timeout {{ $location.Proxy.ReadTimeout }}s; 822 | 823 | {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }} 824 | proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; 825 | {{ else }} 826 | proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; 827 | {{ end }} 828 | proxy_buffering off; 829 | proxy_buffer_size "{{ $location.Proxy.BufferSize }}"; 830 | proxy_buffers 4 "{{ $location.Proxy.BufferSize }}"; 831 | proxy_request_buffering "{{ $location.Proxy.RequestBuffering }}"; 832 | 833 | proxy_http_version 1.1; 834 | 835 | proxy_cookie_domain {{ $location.Proxy.CookieDomain }}; 836 | proxy_cookie_path {{ $location.Proxy.CookiePath }}; 837 | 838 | # In case of errors try the next upstream server before returning an error 839 | proxy_next_upstream {{ buildNextUpstream $location.Proxy.NextUpstream $all.Cfg.RetryNonIdempotent }}; 840 | 841 | {{/* rewrite only works if the content is not compressed */}} 842 | {{ if $location.Rewrite.AddBaseURL }} 843 | proxy_set_header Accept-Encoding ""; 844 | {{ end }} 845 | 846 | {{/* Add any additional configuration defined */}} 847 | {{ $location.ConfigurationSnippet }} 848 | 849 | {{ if not (empty $all.Cfg.LocationSnippet) }} 850 | # Custom code snippet configured in the configuration configmap 851 | {{ $all.Cfg.LocationSnippet }} 852 | {{ end }} 853 | 854 | {{/* if we are sending the request to a custom default backend, we add the required headers */}} 855 | {{ if (hasPrefix $location.Backend "custom-default-backend-") }} 856 | proxy_set_header X-Code 503; 857 | proxy_set_header X-Format $http_accept; 858 | proxy_set_header X-Namespace $namespace; 859 | proxy_set_header X-Ingress-Name $ingress_name; 860 | proxy_set_header X-Service-Name $service_name; 861 | {{ end }} 862 | 863 | 864 | {{ if not (empty $location.Backend) }} 865 | {{ buildProxyPass $server.Hostname $all.Backends $location }} 866 | {{ else }} 867 | # No endpoints available for the request 868 | return 503; 869 | {{ end }} 870 | {{ else }} 871 | # Location denied. Reason: {{ $location.Denied }} 872 | return 503; 873 | {{ end }} 874 | } 875 | 876 | {{ end }} 877 | 878 | {{ if eq $server.Hostname "_" }} 879 | # health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }} 880 | location {{ $all.HealthzURI }} { 881 | access_log off; 882 | return 200; 883 | } 884 | 885 | # this is required to avoid error if nginx is being monitored 886 | # with an external software (like sysdig) 887 | location /nginx_status { 888 | allow 127.0.0.1; 889 | {{ if $all.IsIPV6Enabled }}allow ::1;{{ end }} 890 | deny all; 891 | 892 | access_log off; 893 | stub_status on; 894 | } 895 | 896 | {{ end }} 897 | 898 | {{ end }} 899 | --------------------------------------------------------------------------------