├── files ├── register-to-rancher.sh ├── cloud-config-base.yaml ├── k3s-install.sh ├── rancher-install.sh └── ingress-install.sh ├── .pre-commit-config.yaml ├── .gitignore ├── examples ├── default │ ├── bastion.tmpl │ └── main.tf └── rancher │ ├── bastion.tmpl │ └── main.tf ├── output.tf ├── loadbalancer.tf ├── data.tf ├── main.tf ├── README.md ├── variables.tf ├── infra.tf └── LICENSE /files/register-to-rancher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | %{ if is_k3s_server } 4 | %{ if !install_rancher } 5 | ${registration_command} 6 | %{ endif } 7 | %{ endif } 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: git://github.com/antonbabenko/pre-commit-terraform 2 | rev: v1.19.0 3 | hooks: 4 | - id: terraform_fmt 5 | - id: terraform_docs 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.tfstate* 3 | **/.terraform 4 | **/*.plan 5 | **/secret_files 6 | **/*.tfenvs 7 | **/aws_accounts/iam 8 | .DS_Store 9 | outputs/ 10 | terraform.d/ 11 | **/terraform.tfvars 12 | -------------------------------------------------------------------------------- /files/cloud-config-base.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | %{ if length(ssh_keys) > 0 } 3 | ssh_authorized_keys: 4 | %{ for ssh_key in ssh_keys } 5 | - ${ssh_key} 6 | %{ endfor } 7 | %{ endif } 8 | runcmd: 9 | - apt-get update 10 | - apt-get install -y software-properties-common 11 | - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y 12 | -------------------------------------------------------------------------------- /examples/default/bastion.tmpl: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | %{ if length(ssh_keys) > 0 } 3 | ssh_authorized_keys: 4 | %{ for ssh_key in ssh_keys } 5 | - ${ssh_key} 6 | %{ endfor } 7 | %{ endif } 8 | runcmd: 9 | - apt-get update 10 | - apt-get install -y software-properties-common 11 | - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y 12 | -------------------------------------------------------------------------------- /examples/rancher/bastion.tmpl: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | %{ if length(ssh_keys) > 0 } 3 | ssh_authorized_keys: 4 | %{ for ssh_key in ssh_keys } 5 | - ${ssh_key} 6 | %{ endfor } 7 | %{ endif } 8 | runcmd: 9 | - apt-get update 10 | - apt-get install -y software-properties-common 11 | - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y 12 | -------------------------------------------------------------------------------- /output.tf: -------------------------------------------------------------------------------- 1 | output "rancher_admin_password" { 2 | value = local.rancher_password 3 | sensitive = true 4 | } 5 | 6 | output "rancher_url" { 7 | value = local.install_rancher ? rancher2_bootstrap.admin.0.url : null 8 | } 9 | 10 | output "rancher_token" { 11 | value = local.install_rancher ? rancher2_bootstrap.admin.0.token : null 12 | sensitive = true 13 | } 14 | -------------------------------------------------------------------------------- /files/k3s-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | %{ if is_k3s_server } 4 | %{ if k3s_storage_endpoint != "sqlite" } 5 | curl -o ${k3s_storage_cafile} https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem 6 | %{ endif } 7 | %{ endif } 8 | 9 | until (curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION='v${install_k3s_version}' INSTALL_K3S_EXEC='%{ if is_k3s_server }${k3s_tls_san} ${k3s_disable_agent} ${k3s_deploy_traefik} %{ endif}${k3s_exec}' K3S_CLUSTER_SECRET='${k3s_cluster_secret}' %{ if is_k3s_server }%{ if k3s_storage_endpoint != "sqlite" }K3S_STORAGE_CAFILE='${k3s_storage_cafile}'%{ endif } %{ if k3s_storage_endpoint != "sqlite" }K3S_STORAGE_ENDPOINT='${k3s_storage_endpoint}'%{ endif } %{ endif }%{ if !is_k3s_server } K3S_URL='https://${k3s_url}:6443'%{ endif } sh -); do 10 | echo 'k3s did not install correctly' 11 | sleep 2 12 | done 13 | 14 | %{ if is_k3s_server } 15 | until kubectl get pods -A | grep 'Running'; 16 | do 17 | echo 'Waiting for k3s startup' 18 | sleep 5 19 | done 20 | %{ endif } 21 | -------------------------------------------------------------------------------- /files/rancher-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | %{ if install_certmanager } 3 | kubectl create namespace cert-manager 4 | kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true 5 | sleep 5 6 | kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v${certmanager_version}/cert-manager-no-webhook.yaml 7 | 8 | until [ "$(kubectl get pods --namespace cert-manager |grep Running|wc -l)" = "2" ]; do 9 | sleep 2 10 | done 11 | 12 | %{ if install_rancher } 13 | cat < /var/lib/rancher/k3s/server/manifests/rancher.yaml 14 | --- 15 | apiVersion: v1 16 | kind: Namespace 17 | metadata: 18 | name: cattle-system 19 | --- 20 | apiVersion: helm.cattle.io/v1 21 | kind: HelmChart 22 | metadata: 23 | name: rancher 24 | namespace: kube-system 25 | spec: 26 | chart: https://releases.rancher.com/server-charts/latest/rancher-${rancher_version}.tgz 27 | targetNamespace: cattle-system 28 | valuesContent: |- 29 | hostname: ${rancher_hostname} 30 | ingress: 31 | tls: 32 | source: letsEncrypt 33 | letsEncrypt: 34 | email: ${letsencrypt_email} 35 | EOF 36 | %{ endif } 37 | %{ endif } 38 | -------------------------------------------------------------------------------- /loadbalancer.tf: -------------------------------------------------------------------------------- 1 | resource "random_pet" "lb" {} 2 | 3 | resource "aws_lb" "server-lb" { 4 | name = substr("${local.name}-int-${random_pet.lb.id}", 0, 24) 5 | internal = true 6 | load_balancer_type = "network" 7 | subnets = local.private_subnets 8 | } 9 | 10 | resource "aws_lb_listener" "server-port_6443" { 11 | load_balancer_arn = aws_lb.server-lb.arn 12 | port = "6443" 13 | protocol = "TCP" 14 | 15 | default_action { 16 | type = "forward" 17 | target_group_arn = aws_lb_target_group.server-6443.arn 18 | } 19 | } 20 | 21 | resource "aws_lb_target_group" "server-6443" { 22 | name = substr("${local.name}-6443-${random_pet.lb.id}", 0, 24) 23 | port = 6443 24 | protocol = "TCP" 25 | vpc_id = data.aws_vpc.default.id 26 | } 27 | 28 | 29 | resource "aws_lb" "lb" { 30 | count = local.create_external_nlb 31 | name = substr("${local.name}-ext-${random_pet.lb.id}", 0, 24) 32 | internal = false 33 | load_balancer_type = "network" 34 | subnets = local.public_subnets 35 | 36 | tags = { 37 | "kubernetes.io/cluster/${local.name}" = "" 38 | } 39 | } 40 | 41 | resource "aws_lb_listener" "port_443" { 42 | count = local.create_external_nlb 43 | load_balancer_arn = aws_lb.lb.0.arn 44 | port = "443" 45 | protocol = "TCP" 46 | 47 | default_action { 48 | type = "forward" 49 | target_group_arn = aws_lb_target_group.agent-443.0.arn 50 | } 51 | } 52 | 53 | resource "aws_lb_listener" "port_80" { 54 | count = local.create_external_nlb 55 | load_balancer_arn = aws_lb.lb.0.arn 56 | port = "80" 57 | protocol = "TCP" 58 | 59 | default_action { 60 | type = "forward" 61 | target_group_arn = aws_lb_target_group.agent-80.0.arn 62 | } 63 | } 64 | 65 | resource "aws_lb_target_group" "agent-443" { 66 | count = local.create_external_nlb 67 | name = substr("${local.name}-443-${random_pet.lb.id}", 0, 24) 68 | port = 443 69 | protocol = "TCP" 70 | vpc_id = data.aws_vpc.default.id 71 | 72 | health_check { 73 | interval = 10 74 | timeout = 6 75 | path = "/healthz" 76 | port = 80 77 | protocol = "HTTP" 78 | healthy_threshold = 3 79 | unhealthy_threshold = 3 80 | matcher = "200-399" 81 | } 82 | 83 | tags = { 84 | "kubernetes.io/cluster/${local.name}" = "" 85 | } 86 | } 87 | 88 | resource "aws_lb_target_group" "agent-80" { 89 | count = local.create_external_nlb 90 | name = substr("${local.name}-80-${random_pet.lb.id}", 0, 24) 91 | port = 80 92 | protocol = "TCP" 93 | vpc_id = data.aws_vpc.default.id 94 | 95 | health_check { 96 | interval = 10 97 | timeout = 6 98 | path = "/healthz" 99 | port = 80 100 | protocol = "HTTP" 101 | healthy_threshold = 3 102 | unhealthy_threshold = 3 103 | matcher = "200-399" 104 | } 105 | 106 | tags = { 107 | "kubernetes.io/cluster/${local.name}" = "" 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /data.tf: -------------------------------------------------------------------------------- 1 | data "aws_vpc" "default" { 2 | default = false 3 | id = var.vpc_id 4 | } 5 | 6 | data "aws_subnet_ids" "available" { 7 | vpc_id = data.aws_vpc.default.id 8 | } 9 | 10 | data "aws_route53_zone" "dns_zone" { 11 | provider = aws.r53 12 | name = local.r53_domain 13 | } 14 | 15 | data "aws_ami" "ubuntu" { 16 | most_recent = true 17 | owners = ["099720109477"] 18 | 19 | filter { 20 | name = "name" 21 | values = ["ubuntu-minimal/images/*/ubuntu-bionic-18.04-*"] 22 | } 23 | 24 | filter { 25 | name = "virtualization-type" 26 | values = ["hvm"] 27 | } 28 | 29 | filter { 30 | name = "root-device-type" 31 | values = ["ebs"] 32 | } 33 | 34 | filter { 35 | name = "architecture" 36 | values = ["x86_64"] 37 | } 38 | } 39 | 40 | data "template_cloudinit_config" "k3s_server" { 41 | gzip = true 42 | base64_encode = true 43 | 44 | # Main cloud-config configuration file. 45 | part { 46 | filename = "init.cfg" 47 | content_type = "text/cloud-config" 48 | content = templatefile("${path.module}/files/cloud-config-base.yaml", { ssh_keys = var.ssh_keys }) 49 | } 50 | 51 | part { 52 | content_type = "text/x-shellscript" 53 | content = templatefile("${path.module}/files/k3s-install.sh", { install_k3s_version = local.install_k3s_version, k3s_exec = local.server_k3s_exec, k3s_cluster_secret = local.k3s_cluster_secret, is_k3s_server = true, k3s_url = aws_lb.server-lb.dns_name, k3s_storage_endpoint = local.k3s_storage_endpoint, k3s_storage_cafile = local.k3s_storage_cafile, k3s_disable_agent = local.k3s_disable_agent, k3s_tls_san = local.k3s_tls_san, k3s_deploy_traefik = local.k3s_deploy_traefik }) 54 | } 55 | 56 | part { 57 | content_type = "text/x-shellscript" 58 | content = templatefile("${path.module}/files/ingress-install.sh", { install_nginx_ingress = local.install_nginx_ingress }) 59 | } 60 | 61 | part { 62 | content_type = "text/x-shellscript" 63 | content = templatefile("${path.module}/files/rancher-install.sh", { certmanager_version = local.certmanager_version, letsencrypt_email = local.letsencrypt_email, rancher_version = local.rancher_version, rancher_hostname = "${local.name}.${local.domain}", install_rancher = local.install_rancher, install_nginx_ingress = local.install_nginx_ingress, install_certmanager = local.install_certmanager }) 64 | } 65 | 66 | part { 67 | content_type = "text/x-shellscript" 68 | content = templatefile("${path.module}/files/register-to-rancher.sh", { is_k3s_server = true, install_rancher = local.install_rancher, registration_command = local.registration_command }) 69 | } 70 | } 71 | 72 | data "template_cloudinit_config" "k3s_agent" { 73 | gzip = true 74 | base64_encode = true 75 | 76 | # Main cloud-config configuration file. 77 | part { 78 | filename = "init.cfg" 79 | content_type = "text/cloud-config" 80 | content = templatefile("${path.module}/files/cloud-config-base.yaml", { ssh_keys = var.ssh_keys }) 81 | } 82 | 83 | part { 84 | content_type = "text/x-shellscript" 85 | content = templatefile("${path.module}/files/k3s-install.sh", { install_k3s_version = local.install_k3s_version, k3s_exec = local.agent_k3s_exec, k3s_cluster_secret = local.k3s_cluster_secret, is_k3s_server = false, k3s_url = aws_lb.server-lb.dns_name, k3s_storage_endpoint = local.k3s_storage_endpoint, k3s_storage_cafile = local.k3s_storage_cafile, k3s_disable_agent = local.k3s_disable_agent, k3s_tls_san = local.k3s_tls_san, k3s_deploy_traefik = local.k3s_deploy_traefik }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /examples/rancher/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-west-2" 3 | profile = "rancher-eng" 4 | } 5 | 6 | provider "aws" { 7 | alias = "r53" 8 | region = "us-west-2" 9 | profile = "rancher-eng" 10 | } 11 | 12 | module "vpc" { 13 | source = "terraform-aws-modules/vpc/aws" 14 | version = "2.17.0" 15 | 16 | name = "example" 17 | cidr = "10.105.0.0/16" 18 | 19 | azs = ["us-west-2a", "us-west-2b", "us-west-2c"] 20 | public_subnets = ["10.105.1.0/24", "10.105.2.0/24", "10.105.3.0/24"] 21 | private_subnets = ["10.105.4.0/24", "10.105.5.0/24", "10.105.6.0/24"] 22 | 23 | create_database_subnet_group = false 24 | 25 | enable_dns_hostnames = true 26 | enable_dns_support = true 27 | enable_nat_gateway = true 28 | 29 | tags = { 30 | "Name" = "example" 31 | } 32 | } 33 | 34 | data "aws_ami" "ubuntu" { 35 | most_recent = true 36 | owners = ["099720109477"] 37 | 38 | filter { 39 | name = "name" 40 | values = ["ubuntu-minimal/images/*/ubuntu-bionic-18.04-*"] 41 | } 42 | 43 | filter { 44 | name = "virtualization-type" 45 | values = ["hvm"] 46 | } 47 | 48 | filter { 49 | name = "root-device-type" 50 | values = ["ebs"] 51 | } 52 | 53 | filter { 54 | name = "architecture" 55 | values = ["x86_64"] 56 | } 57 | } 58 | 59 | resource "aws_security_group" "bastion" { 60 | name = "example-bastion" 61 | vpc_id = module.vpc.vpc_id 62 | } 63 | 64 | resource "aws_security_group_rule" "bastion_ssh" { 65 | type = "ingress" 66 | from_port = 22 67 | to_port = 22 68 | protocol = "TCP" 69 | cidr_blocks = ["0.0.0.0/0"] 70 | security_group_id = aws_security_group.bastion.id 71 | } 72 | 73 | resource "aws_security_group_rule" "bastion_egress_all" { 74 | type = "egress" 75 | from_port = 0 76 | to_port = 0 77 | protocol = "-1" 78 | cidr_blocks = ["0.0.0.0/0"] 79 | security_group_id = aws_security_group.bastion.id 80 | } 81 | 82 | resource "aws_instance" "bastion" { 83 | ami = "${data.aws_ami.ubuntu.id}" 84 | instance_type = "t2.micro" 85 | subnet_id = element(module.vpc.public_subnets, 0) 86 | user_data = templatefile("${path.module}/bastion.tmpl", { ssh_keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak"] }) 87 | 88 | vpc_security_group_ids = [aws_security_group.bastion.id, module.vpc.default_security_group_id] 89 | 90 | tags = { 91 | Name = "example-bastion" 92 | } 93 | } 94 | 95 | module "k3s_rancher" { 96 | source = "../../" 97 | vpc_id = module.vpc.vpc_id 98 | aws_region = "us-west-2" 99 | aws_profile = "rancher-eng" 100 | rancher_password = "u7qmyhm3wbgujjuijs3rqfpm2e" 101 | install_rancher = true 102 | install_certmanager = true 103 | install_nginx_ingress = true 104 | k3s_deploy_traefik = false 105 | private_subnets = module.vpc.private_subnets 106 | public_subnets = module.vpc.public_subnets 107 | ssh_keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak"] 108 | name = "example" 109 | k3s_cluster_secret = "secretvaluechangeme" 110 | domain = "eng.rancher.space" 111 | aws_azs = ["us-west-2a", "us-west-2b", "us-west-2c"] 112 | k3s_storage_endpoint = "postgres" 113 | db_user = "exampleuser" 114 | db_pass = "mD,50cbf5597fd320b6a732ce778082a0359" 115 | extra_server_security_groups = [module.vpc.default_security_group_id] 116 | extra_agent_security_groups = [module.vpc.default_security_group_id] 117 | private_subnets_cidr_blocks = module.vpc.private_subnets_cidr_blocks 118 | providers = { 119 | aws = "aws" 120 | aws.r53 = "aws.r53" 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /examples/default/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-west-2" 3 | profile = "rancher-eng" 4 | } 5 | 6 | provider "aws" { 7 | alias = "r53" 8 | region = "us-west-2" 9 | profile = "rancher-eng" 10 | } 11 | 12 | module "vpc" { 13 | source = "terraform-aws-modules/vpc/aws" 14 | version = "2.17.0" 15 | 16 | name = "example" 17 | cidr = "10.105.0.0/16" 18 | 19 | azs = ["us-west-2a", "us-west-2b", "us-west-2c"] 20 | public_subnets = ["10.105.1.0/24", "10.105.2.0/24", "10.105.3.0/24"] 21 | private_subnets = ["10.105.4.0/24", "10.105.5.0/24", "10.105.6.0/24"] 22 | 23 | create_database_subnet_group = false 24 | 25 | enable_dns_hostnames = true 26 | enable_dns_support = true 27 | enable_nat_gateway = true 28 | 29 | tags = { 30 | "Name" = "example" 31 | } 32 | } 33 | 34 | data "aws_ami" "ubuntu" { 35 | most_recent = true 36 | owners = ["099720109477"] 37 | 38 | filter { 39 | name = "name" 40 | values = ["ubuntu-minimal/images/*/ubuntu-bionic-18.04-*"] 41 | } 42 | 43 | filter { 44 | name = "virtualization-type" 45 | values = ["hvm"] 46 | } 47 | 48 | filter { 49 | name = "root-device-type" 50 | values = ["ebs"] 51 | } 52 | 53 | filter { 54 | name = "architecture" 55 | values = ["x86_64"] 56 | } 57 | } 58 | 59 | resource "aws_security_group" "bastion" { 60 | name = "example-bastion" 61 | vpc_id = module.vpc.vpc_id 62 | } 63 | 64 | resource "aws_security_group_rule" "bastion_ssh" { 65 | type = "ingress" 66 | from_port = 22 67 | to_port = 22 68 | protocol = "TCP" 69 | cidr_blocks = ["0.0.0.0/0"] 70 | security_group_id = aws_security_group.bastion.id 71 | } 72 | 73 | resource "aws_security_group_rule" "bastion_egress_all" { 74 | type = "egress" 75 | from_port = 0 76 | to_port = 0 77 | protocol = "-1" 78 | cidr_blocks = ["0.0.0.0/0"] 79 | security_group_id = aws_security_group.bastion.id 80 | } 81 | 82 | resource "aws_instance" "bastion" { 83 | ami = "${data.aws_ami.ubuntu.id}" 84 | instance_type = "t2.micro" 85 | subnet_id = element(module.vpc.public_subnets, 0) 86 | user_data = templatefile("${path.module}/bastion.tmpl", { ssh_keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak"] }) 87 | 88 | vpc_security_group_ids = [aws_security_group.bastion.id, module.vpc.default_security_group_id] 89 | 90 | tags = { 91 | Name = "example-bastion" 92 | } 93 | } 94 | 95 | provider "rancher2" { 96 | api_url = "https://example.eng.rancher.space" 97 | token_key = "token-4hdgv:zsgmrtqhzf4rf5l7tp6vv6fpxv8jwdxntwsk2bq7mwgmbv8kcg5lsf" 98 | } 99 | 100 | resource "rancher2_cluster" "k3s" { 101 | name = "example-imported" 102 | } 103 | 104 | module "k3s_rancher" { 105 | source = "../../" 106 | vpc_id = module.vpc.vpc_id 107 | aws_region = "us-west-2" 108 | aws_profile = "rancher-eng" 109 | private_subnets = module.vpc.private_subnets 110 | public_subnets = module.vpc.public_subnets 111 | ssh_keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak"] 112 | name = "example" 113 | k3s_cluster_secret = "secretvaluechangeme" 114 | domain = "eng.rancher.space" 115 | aws_azs = ["us-west-2a", "us-west-2b", "us-west-2c"] 116 | k3s_storage_endpoint = "postgres" 117 | db_user = "exampleuser" 118 | db_pass = "mD,50cbf5597fd320b6a732ce778082a0359" 119 | extra_server_security_groups = [module.vpc.default_security_group_id] 120 | extra_agent_security_groups = [module.vpc.default_security_group_id] 121 | private_subnets_cidr_blocks = module.vpc.private_subnets_cidr_blocks 122 | registration_command = rancher2_cluster.k3s.cluster_registration_token[0].command 123 | providers = { 124 | aws = "aws" 125 | aws.r53 = "aws.r53" 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | rancher2 = ">= 1.6.0" 4 | } 5 | } 6 | 7 | 8 | provider "aws" {} 9 | 10 | provider "aws" { 11 | alias = "r53" 12 | } 13 | 14 | locals { 15 | name = var.name 16 | install_k3s_version = var.install_k3s_version 17 | k3s_cluster_secret = var.k3s_cluster_secret != null ? var.k3s_cluster_secret : random_password.k3s_cluster_secret.result 18 | server_instance_type = var.server_instance_type 19 | agent_instance_type = var.agent_instance_type 20 | agent_image_id = var.agent_image_id != null ? var.agent_image_id : data.aws_ami.ubuntu.id 21 | server_image_id = var.server_image_id != null ? var.server_image_id : data.aws_ami.ubuntu.id 22 | aws_azs = var.aws_azs 23 | public_subnets = length(var.public_subnets) > 0 ? var.public_subnets : data.aws_subnet_ids.available.ids 24 | private_subnets = length(var.private_subnets) > 0 ? var.private_subnets : data.aws_subnet_ids.available.ids 25 | server_node_count = var.server_node_count 26 | agent_node_count = var.agent_node_count 27 | ssh_keys = var.ssh_keys 28 | deploy_rds = var.k3s_storage_endpoint != "sqlite" ? 1 : 0 29 | db_instance_type = var.db_instance_type 30 | db_user = var.db_user 31 | db_pass = var.db_pass 32 | db_name = var.db_name != null ? var.db_name : var.name 33 | db_node_count = var.k3s_storage_endpoint != "sqlite" ? var.db_node_count : 0 34 | k3s_storage_cafile = var.k3s_storage_cafile 35 | k3s_storage_endpoint = var.k3s_storage_endpoint == "sqlite" ? null : "postgres://${local.db_user}:${local.db_pass}@${aws_rds_cluster.k3s.0.endpoint}/${local.db_name}" 36 | k3s_disable_agent = var.k3s_disable_agent ? "--disable-agent" : "" 37 | k3s_tls_san = var.k3s_tls_san != null ? var.k3s_tls_san : "--tls-san ${aws_lb.server-lb.dns_name}" 38 | k3s_deploy_traefik = var.k3s_deploy_traefik ? "" : "--no-deploy traefik" 39 | server_k3s_exec = "" 40 | agent_k3s_exec = "" 41 | certmanager_version = var.certmanager_version 42 | rancher_version = var.rancher_version 43 | letsencrypt_email = var.letsencrypt_email 44 | domain = var.domain 45 | r53_domain = length(var.r53_domain) > 0 ? var.r53_domain : local.domain 46 | private_subnets_cidr_blocks = var.private_subnets_cidr_blocks 47 | public_subnets_cidr_blocks = var.public_subnets_cidr_blocks 48 | skip_final_snapshot = var.skip_final_snapshot 49 | install_certmanager = var.install_certmanager 50 | install_rancher = var.install_rancher 51 | install_nginx_ingress = var.install_nginx_ingress 52 | create_external_nlb = var.create_external_nlb ? 1 : 0 53 | registration_command = var.registration_command 54 | rancher_password = var.rancher_password 55 | } 56 | 57 | resource "random_password" "k3s_cluster_secret" { 58 | length = 30 59 | special = false 60 | } 61 | 62 | provider "rancher2" { 63 | alias = "bootstrap" 64 | api_url = "https://${local.name}.${local.domain}" 65 | bootstrap = true 66 | } 67 | 68 | resource "null_resource" "wait_for_rancher" { 69 | count = local.install_rancher ? 1 : 0 70 | provisioner "local-exec" { 71 | command = <&1 | grep "subject:") 74 | echo "Cert Subject Response: $${subject}" 75 | if [ "$${subject}" != "* subject: CN=$${RANCHER_HOSTNAME}" ]; then 76 | sleep 10 77 | fi 78 | done 79 | while [ "$${resp}" != "pong" ]; do 80 | resp=$(curl -sSk -m 2 "https://$${RANCHER_HOSTNAME}/ping") 81 | echo "Rancher Response: $${resp}" 82 | if [ "$${resp}" != "pong" ]; then 83 | sleep 10 84 | fi 85 | done 86 | EOF 87 | 88 | 89 | environment = { 90 | RANCHER_HOSTNAME = "${local.name}.${local.domain}" 91 | } 92 | } 93 | depends_on = [ 94 | aws_autoscaling_group.k3s_server, 95 | aws_autoscaling_group.k3s_agent, 96 | aws_rds_cluster_instance.k3s, 97 | aws_route53_record.rancher 98 | ] 99 | } 100 | 101 | resource "rancher2_bootstrap" "admin" { 102 | count = local.install_rancher ? 1 : 0 103 | provider = rancher2.bootstrap 104 | password = local.rancher_password 105 | depends_on = [null_resource.wait_for_rancher] 106 | } 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Terraform K3S AWS Cluster 2 | 3 | This module supports creating a k3s cluster with a postgres backend in AWS. It allows you to optionally install nginx-ingress, Rancher Server, and cert-manager, or import your K3S cluster into an existing Rancher Server. 4 | 5 | 6 | ## Inputs 7 | 8 | | Name | Description | Type | Default | Required | 9 | |------|-------------|:----:|:-----:|:-----:| 10 | | agent\_image\_id | AMI to use for k3s agent instances | string | `"null"` | no | 11 | | agent\_instance\_ssh\_user | Username for sshing into instances | string | `"ubuntu"` | no | 12 | | agent\_instance\_type | | string | `"m5.large"` | no | 13 | | agent\_node\_count | Number of agent nodes to launch | number | `"3"` | no | 14 | | aws\_azs | List of AWS Availability Zones in the VPC | list | `"null"` | no | 15 | | aws\_profile | Name of the AWS Profile to use for authentication | string | `"null"` | no | 16 | | aws\_region | | string | `"null"` | no | 17 | | certmanager\_version | Version of cert-manager to install | string | `"0.9.1"` | no | 18 | | create\_external\_nlb | Boolean that defines whether or not to create an external load balancer | bool | `"true"` | no | 19 | | db\_instance\_type | | string | `"db.r5.large"` | no | 20 | | db\_name | Name of database to create in RDS | string | `"null"` | no | 21 | | db\_node\_count | Number of RDS database instances to launch | number | `"1"` | no | 22 | | db\_pass | Password for RDS user | string | n/a | yes | 23 | | db\_user | Username for RDS database | string | n/a | yes | 24 | | domain | | string | `"eng.rancher.space"` | no | 25 | | extra\_agent\_security\_groups | Additional security groups to attach to k3s agent instances | list | `[]` | no | 26 | | extra\_server\_security\_groups | Additional security groups to attach to k3s server instances | list | `[]` | no | 27 | | install\_certmanager | Boolean that defines whether or not to install Cert-Manager | bool | `"false"` | no | 28 | | install\_k3s\_version | Version of K3S to install | string | `"0.9.1"` | no | 29 | | install\_nginx\_ingress | Boolean that defines whether or not to install nginx-ingress | bool | `"false"` | no | 30 | | install\_rancher | Boolean that defines whether or not to install Rancher | bool | `"false"` | no | 31 | | k3s\_cluster\_secret | Override to set k3s cluster registration secret | string | `"null"` | no | 32 | | k3s\_deploy\_traefik | Configures whether to deploy traefik ingress or not | bool | `"true"` | no | 33 | | k3s\_disable\_agent | Whether to run the k3s agent on the same host as the k3s server | bool | `"false"` | no | 34 | | k3s\_storage\_cafile | Location to download RDS CA Bundle | string | `"/srv/rds-combined-ca-bundle.pem"` | no | 35 | | k3s\_storage\_endpoint | Storage Backend for K3S cluster to use. Valid options are 'sqlite' or 'postgres' | string | `"sqlite"` | no | 36 | | k3s\_tls\_san | Sets k3s tls-san flag to this value instead of the default load balancer | string | `"null"` | no | 37 | | letsencrypt\_email | LetsEncrypt email address to use | string | `"none@none.com"` | no | 38 | | name | Name for deployment | string | `"rancher-demo"` | no | 39 | | private\_subnets | List of private subnet ids. | list | `[]` | no | 40 | | private\_subnets\_cidr\_blocks | List of cidr_blocks of private subnets | list | `[]` | no | 41 | | public\_subnets | List of public subnet ids. | list | `[]` | no | 42 | | public\_subnets\_cidr\_blocks | List of cidr_blocks of public subnets | list | `[]` | no | 43 | | r53\_domain | DNS domain for Route53 zone (defaults to domain if unset) | string | `""` | no | 44 | | rancher2\_token\_key | Rancher2 API token for authentication | string | `"null"` | no | 45 | | rancher\_chart | Helm chart to use for Rancher install | string | `"rancher-stable/rancher"` | no | 46 | | rancher\_password | Password to set for admin user during bootstrap of Rancher Server | string | `""` | no | 47 | | rancher\_version | Version of Rancher to install | string | `"2.3.1"` | no | 48 | | registration\_command | Registration command to import cluster into Rancher. Should not be used when installing Rancher in this same cluster | string | `""` | no | 49 | | server\_image\_id | AMI to use for k3s server instances | string | `"null"` | no | 50 | | server\_instance\_ssh\_user | Username for sshing into instances | string | `"ubuntu"` | no | 51 | | server\_instance\_type | | string | `"m5.large"` | no | 52 | | server\_node\_count | Number of server nodes to launch | number | `"1"` | no | 53 | | skip\_final\_snapshot | Boolean that defines whether or not the final snapshot should be created on RDS cluster deletion | bool | `"true"` | no | 54 | | ssh\_keys | SSH keys to inject into Rancher instances | list | `[]` | no | 55 | | vpc\_id | The vpc id that Rancher should use | string | `"null"` | no | 56 | 57 | ## Outputs 58 | 59 | | Name | Description | 60 | |------|-------------| 61 | | rancher\_admin\_password | | 62 | | rancher\_token | | 63 | | rancher\_url | | 64 | 65 | 66 | 67 | # License 68 | 69 | Copyright (c) 2014-2019 [Rancher Labs, Inc.](http://rancher.com) 70 | 71 | Licensed under the Apache License, Version 2.0 (the "License"); 72 | you may not use this file except in compliance with the License. 73 | You may obtain a copy of the License at 74 | 75 | [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 76 | 77 | Unless required by applicable law or agreed to in writing, software 78 | distributed under the License is distributed on an "AS IS" BASIS, 79 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 80 | See the License for the specific language governing permissions and 81 | limitations under the License. 82 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "rancher_password" { 2 | type = string 3 | default = "" 4 | description = "Password to set for admin user during bootstrap of Rancher Server" 5 | } 6 | 7 | variable "rancher_version" { 8 | type = string 9 | default = "2.3.1" 10 | description = "Version of Rancher to install" 11 | } 12 | 13 | variable "agent_image_id" { 14 | type = string 15 | default = null 16 | description = "AMI to use for k3s agent instances" 17 | } 18 | 19 | variable "server_image_id" { 20 | type = string 21 | default = null 22 | description = "AMI to use for k3s server instances" 23 | } 24 | 25 | variable "ssh_keys" { 26 | type = list 27 | default = [] 28 | description = "SSH keys to inject into Rancher instances" 29 | } 30 | 31 | variable "rancher_chart" { 32 | type = string 33 | default = "rancher-stable/rancher" 34 | description = "Helm chart to use for Rancher install" 35 | } 36 | 37 | variable "name" { 38 | type = string 39 | default = "rancher-demo" 40 | description = "Name for deployment" 41 | } 42 | 43 | variable "letsencrypt_email" { 44 | type = string 45 | default = "none@none.com" 46 | description = "LetsEncrypt email address to use" 47 | } 48 | 49 | variable "domain" { 50 | type = string 51 | default = "eng.rancher.space" 52 | } 53 | 54 | variable "r53_domain" { 55 | type = string 56 | default = "" 57 | description = "DNS domain for Route53 zone (defaults to domain if unset)" 58 | } 59 | 60 | variable "server_instance_type" { 61 | type = string 62 | default = "m5.large" 63 | } 64 | 65 | variable "agent_instance_type" { 66 | type = string 67 | default = "m5.large" 68 | } 69 | 70 | variable "server_node_count" { 71 | type = number 72 | default = 1 73 | description = "Number of server nodes to launch" 74 | } 75 | 76 | variable "agent_node_count" { 77 | type = number 78 | default = 3 79 | description = "Number of agent nodes to launch" 80 | } 81 | 82 | variable "db_node_count" { 83 | type = number 84 | default = 1 85 | description = "Number of RDS database instances to launch" 86 | } 87 | 88 | variable "server_instance_ssh_user" { 89 | type = string 90 | default = "ubuntu" 91 | description = "Username for sshing into instances" 92 | } 93 | 94 | variable "agent_instance_ssh_user" { 95 | type = string 96 | default = "ubuntu" 97 | description = "Username for sshing into instances" 98 | } 99 | 100 | variable "certmanager_version" { 101 | type = string 102 | default = "0.9.1" 103 | description = "Version of cert-manager to install" 104 | } 105 | 106 | variable "vpc_id" { 107 | type = string 108 | default = null 109 | description = "The vpc id that Rancher should use" 110 | } 111 | 112 | variable "aws_region" { 113 | type = string 114 | default = null 115 | } 116 | 117 | variable "aws_profile" { 118 | type = string 119 | default = null 120 | description = "Name of the AWS Profile to use for authentication" 121 | } 122 | 123 | variable "public_subnets" { 124 | default = [] 125 | type = list 126 | description = "List of public subnet ids." 127 | } 128 | 129 | variable "private_subnets" { 130 | default = [] 131 | type = list 132 | description = "List of private subnet ids." 133 | } 134 | 135 | variable "install_k3s_version" { 136 | default = "0.9.1" 137 | type = string 138 | description = "Version of K3S to install" 139 | } 140 | 141 | variable "k3s_cluster_secret" { 142 | default = null 143 | type = string 144 | description = "Override to set k3s cluster registration secret" 145 | } 146 | 147 | variable "extra_server_security_groups" { 148 | default = [] 149 | type = list 150 | description = "Additional security groups to attach to k3s server instances" 151 | } 152 | 153 | variable "extra_agent_security_groups" { 154 | default = [] 155 | type = list 156 | description = "Additional security groups to attach to k3s agent instances" 157 | } 158 | 159 | variable "aws_azs" { 160 | default = null 161 | type = list 162 | description = "List of AWS Availability Zones in the VPC" 163 | } 164 | 165 | variable "db_instance_type" { 166 | default = "db.r5.large" 167 | } 168 | 169 | variable "db_name" { 170 | default = null 171 | type = string 172 | description = "Name of database to create in RDS" 173 | } 174 | 175 | variable "db_user" { 176 | type = string 177 | description = "Username for RDS database" 178 | } 179 | 180 | variable "db_pass" { 181 | type = string 182 | description = "Password for RDS user" 183 | } 184 | 185 | variable "private_subnets_cidr_blocks" { 186 | default = [] 187 | type = list 188 | description = "List of cidr_blocks of private subnets" 189 | } 190 | 191 | variable "public_subnets_cidr_blocks" { 192 | default = [] 193 | type = list 194 | description = "List of cidr_blocks of public subnets" 195 | } 196 | 197 | variable "skip_final_snapshot" { 198 | default = true 199 | type = bool 200 | description = "Boolean that defines whether or not the final snapshot should be created on RDS cluster deletion" 201 | } 202 | 203 | variable "install_rancher" { 204 | default = false 205 | type = bool 206 | description = "Boolean that defines whether or not to install Rancher" 207 | } 208 | 209 | variable "install_nginx_ingress" { 210 | default = false 211 | type = bool 212 | description = "Boolean that defines whether or not to install nginx-ingress" 213 | } 214 | 215 | variable "install_certmanager" { 216 | default = false 217 | type = bool 218 | description = "Boolean that defines whether or not to install Cert-Manager" 219 | } 220 | 221 | variable "create_external_nlb" { 222 | default = true 223 | type = bool 224 | description = "Boolean that defines whether or not to create an external load balancer" 225 | } 226 | 227 | variable "k3s_storage_cafile" { 228 | default = "/srv/rds-combined-ca-bundle.pem" 229 | type = string 230 | description = "Location to download RDS CA Bundle" 231 | } 232 | 233 | variable "registration_command" { 234 | default = "" 235 | type = string 236 | description = "Registration command to import cluster into Rancher. Should not be used when installing Rancher in this same cluster" 237 | } 238 | 239 | variable "k3s_storage_endpoint" { 240 | default = "sqlite" 241 | type = string 242 | description = "Storage Backend for K3S cluster to use. Valid options are 'sqlite' or 'postgres'" 243 | } 244 | 245 | variable "k3s_disable_agent" { 246 | default = false 247 | type = bool 248 | description = "Whether to run the k3s agent on the same host as the k3s server" 249 | } 250 | 251 | variable "k3s_tls_san" { 252 | default = null 253 | type = string 254 | description = "Sets k3s tls-san flag to this value instead of the default load balancer" 255 | } 256 | 257 | variable "k3s_deploy_traefik" { 258 | default = true 259 | type = bool 260 | description = "Configures whether to deploy traefik ingress or not" 261 | } 262 | 263 | variable "rancher2_token_key" { 264 | default = null 265 | type = string 266 | description = "Rancher2 API token for authentication" 267 | } 268 | -------------------------------------------------------------------------------- /files/ingress-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | %{ if install_nginx_ingress } 3 | cat <<'EOF' > /var/lib/rancher/k3s/server/manifests/ingress.yaml 4 | --- 5 | apiVersion: v1 6 | kind: Namespace 7 | metadata: 8 | name: ingress-nginx 9 | --- 10 | kind: ConfigMap 11 | apiVersion: v1 12 | metadata: 13 | name: nginx-configuration 14 | namespace: ingress-nginx 15 | labels: 16 | app: ingress-nginx 17 | --- 18 | kind: ConfigMap 19 | apiVersion: v1 20 | metadata: 21 | name: tcp-services 22 | namespace: ingress-nginx 23 | --- 24 | kind: ConfigMap 25 | apiVersion: v1 26 | metadata: 27 | name: udp-services 28 | namespace: ingress-nginx 29 | --- 30 | apiVersion: v1 31 | kind: ServiceAccount 32 | metadata: 33 | name: nginx-ingress-serviceaccount 34 | namespace: ingress-nginx 35 | --- 36 | apiVersion: rbac.authorization.k8s.io/v1beta1 37 | kind: ClusterRole 38 | metadata: 39 | name: nginx-ingress-clusterrole 40 | rules: 41 | - apiGroups: 42 | - "" 43 | resources: 44 | - configmaps 45 | - endpoints 46 | - nodes 47 | - pods 48 | - secrets 49 | verbs: 50 | - list 51 | - watch 52 | - apiGroups: 53 | - "" 54 | resources: 55 | - nodes 56 | verbs: 57 | - get 58 | - apiGroups: 59 | - "" 60 | resources: 61 | - services 62 | verbs: 63 | - get 64 | - list 65 | - watch 66 | - apiGroups: 67 | - "extensions" 68 | resources: 69 | - ingresses 70 | - daemonsets 71 | verbs: 72 | - get 73 | - list 74 | - watch 75 | - apiGroups: 76 | - "" 77 | resources: 78 | - events 79 | verbs: 80 | - create 81 | - patch 82 | - apiGroups: 83 | - "extensions" 84 | resources: 85 | - ingresses/status 86 | verbs: 87 | - update 88 | --- 89 | apiVersion: rbac.authorization.k8s.io/v1beta1 90 | kind: Role 91 | metadata: 92 | name: nginx-ingress-role 93 | namespace: ingress-nginx 94 | rules: 95 | - apiGroups: 96 | - "" 97 | resources: 98 | - configmaps 99 | - pods 100 | - secrets 101 | - namespaces 102 | verbs: 103 | - get 104 | - apiGroups: 105 | - "" 106 | resources: 107 | - configmaps 108 | resourceNames: 109 | # Defaults to "-" 110 | # Here: "-" 111 | # This has to be adapted if you change either parameter 112 | # when launching the nginx-ingress-controller. 113 | - "ingress-controller-leader-nginx" 114 | verbs: 115 | - get 116 | - update 117 | - apiGroups: 118 | - "" 119 | resources: 120 | - configmaps 121 | verbs: 122 | - create 123 | - apiGroups: 124 | - "" 125 | resources: 126 | - endpoints 127 | verbs: 128 | - get 129 | --- 130 | apiVersion: rbac.authorization.k8s.io/v1beta1 131 | kind: RoleBinding 132 | metadata: 133 | name: nginx-ingress-role-nisa-binding 134 | namespace: ingress-nginx 135 | roleRef: 136 | apiGroup: rbac.authorization.k8s.io 137 | kind: Role 138 | name: nginx-ingress-role 139 | subjects: 140 | - kind: ServiceAccount 141 | name: nginx-ingress-serviceaccount 142 | namespace: ingress-nginx 143 | --- 144 | apiVersion: rbac.authorization.k8s.io/v1beta1 145 | kind: ClusterRoleBinding 146 | metadata: 147 | name: nginx-ingress-clusterrole-nisa-binding 148 | roleRef: 149 | apiGroup: rbac.authorization.k8s.io 150 | kind: ClusterRole 151 | name: nginx-ingress-clusterrole 152 | subjects: 153 | - kind: ServiceAccount 154 | name: nginx-ingress-serviceaccount 155 | namespace: ingress-nginx 156 | --- 157 | apiVersion: apps/v1 158 | kind: DaemonSet 159 | metadata: 160 | name: nginx-ingress-controller 161 | namespace: ingress-nginx 162 | spec: 163 | selector: 164 | matchLabels: 165 | app: ingress-nginx 166 | template: 167 | metadata: 168 | labels: 169 | app: ingress-nginx 170 | annotations: 171 | prometheus.io/port: '10254' 172 | prometheus.io/scrape: 'true' 173 | spec: 174 | affinity: 175 | nodeAffinity: 176 | requiredDuringSchedulingIgnoredDuringExecution: 177 | nodeSelectorTerms: 178 | - matchExpressions: 179 | - key: beta.kubernetes.io/os 180 | operator: NotIn 181 | values: 182 | - windows 183 | hostNetwork: true 184 | serviceAccountName: nginx-ingress-serviceaccount 185 | tolerations: 186 | - effect: NoExecute 187 | operator: Exists 188 | - effect: NoSchedule 189 | operator: Exists 190 | containers: 191 | - name: nginx-ingress-controller 192 | image: rancher/nginx-ingress-controller:0.21.0-rancher3 193 | args: 194 | - /nginx-ingress-controller 195 | - --default-backend-service=$(POD_NAMESPACE)/default-http-backend 196 | - --configmap=$(POD_NAMESPACE)/nginx-configuration 197 | - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services 198 | - --udp-services-configmap=$(POD_NAMESPACE)/udp-services 199 | - --annotations-prefix=nginx.ingress.kubernetes.io 200 | securityContext: 201 | capabilities: 202 | drop: 203 | - ALL 204 | add: 205 | - NET_BIND_SERVICE 206 | runAsUser: 33 207 | env: 208 | - name: POD_NAME 209 | valueFrom: 210 | fieldRef: 211 | fieldPath: metadata.name 212 | - name: POD_NAMESPACE 213 | valueFrom: 214 | fieldRef: 215 | fieldPath: metadata.namespace 216 | ports: 217 | - name: http 218 | containerPort: 80 219 | - name: https 220 | containerPort: 443 221 | livenessProbe: 222 | failureThreshold: 3 223 | httpGet: 224 | path: /healthz 225 | port: 10254 226 | scheme: HTTP 227 | initialDelaySeconds: 10 228 | periodSeconds: 10 229 | successThreshold: 1 230 | timeoutSeconds: 1 231 | readinessProbe: 232 | failureThreshold: 3 233 | httpGet: 234 | path: /healthz 235 | port: 10254 236 | scheme: HTTP 237 | periodSeconds: 10 238 | successThreshold: 1 239 | timeoutSeconds: 1 240 | --- 241 | apiVersion: apps/v1 242 | kind: Deployment 243 | metadata: 244 | name: default-http-backend 245 | labels: 246 | app: default-http-backend 247 | namespace: ingress-nginx 248 | spec: 249 | replicas: 1 250 | selector: 251 | matchLabels: 252 | app: default-http-backend 253 | template: 254 | metadata: 255 | labels: 256 | app: default-http-backend 257 | spec: 258 | affinity: 259 | nodeAffinity: 260 | requiredDuringSchedulingIgnoredDuringExecution: 261 | nodeSelectorTerms: 262 | - matchExpressions: 263 | - key: beta.kubernetes.io/os 264 | operator: NotIn 265 | values: 266 | - windows 267 | - key: node-role.kubernetes.io/worker 268 | operator: Exists 269 | terminationGracePeriodSeconds: 60 270 | tolerations: 271 | - effect: NoExecute 272 | operator: Exists 273 | - effect: NoSchedule 274 | operator: Exists 275 | containers: 276 | - name: default-http-backend 277 | # Any image is permissable as long as: 278 | # 1. It serves a 404 page at / 279 | # 2. It serves 200 on a /healthz endpoint 280 | image: rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1 281 | livenessProbe: 282 | httpGet: 283 | path: /healthz 284 | port: 8080 285 | scheme: HTTP 286 | initialDelaySeconds: 30 287 | timeoutSeconds: 5 288 | ports: 289 | - containerPort: 8080 290 | resources: 291 | limits: 292 | cpu: 10m 293 | memory: 20Mi 294 | requests: 295 | cpu: 10m 296 | memory: 20Mi 297 | --- 298 | apiVersion: v1 299 | kind: Service 300 | metadata: 301 | name: default-http-backend 302 | namespace: ingress-nginx 303 | labels: 304 | app: default-http-backend 305 | spec: 306 | ports: 307 | - port: 80 308 | targetPort: 8080 309 | selector: 310 | app: default-http-backend 311 | EOF 312 | %{ endif } 313 | -------------------------------------------------------------------------------- /infra.tf: -------------------------------------------------------------------------------- 1 | ############################# 2 | ### Access Control 3 | ############################# 4 | 5 | resource "aws_security_group" "ingress" { 6 | name = "${local.name}-ingress" 7 | vpc_id = data.aws_vpc.default.id 8 | } 9 | 10 | resource "aws_security_group_rule" "ingress_http" { 11 | type = "ingress" 12 | from_port = 80 13 | to_port = 80 14 | protocol = "TCP" 15 | cidr_blocks = ["0.0.0.0/0"] 16 | security_group_id = aws_security_group.ingress.id 17 | } 18 | 19 | resource "aws_security_group_rule" "ingress_https" { 20 | type = "ingress" 21 | from_port = 443 22 | to_port = 443 23 | protocol = "TCP" 24 | cidr_blocks = ["0.0.0.0/0"] 25 | security_group_id = aws_security_group.ingress.id 26 | } 27 | 28 | resource "aws_security_group_rule" "ingress_self" { 29 | type = "ingress" 30 | from_port = 0 31 | to_port = 0 32 | protocol = "-1" 33 | self = true 34 | security_group_id = aws_security_group.ingress.id 35 | } 36 | 37 | resource "aws_security_group_rule" "ingress_egress_all" { 38 | type = "egress" 39 | from_port = 0 40 | to_port = 0 41 | protocol = "-1" 42 | cidr_blocks = ["0.0.0.0/0"] 43 | security_group_id = aws_security_group.ingress.id 44 | } 45 | 46 | resource "aws_security_group" "self" { 47 | name = "${local.name}-self" 48 | vpc_id = data.aws_vpc.default.id 49 | } 50 | 51 | resource "aws_security_group_rule" "self_self" { 52 | type = "ingress" 53 | from_port = 0 54 | to_port = 0 55 | protocol = "-1" 56 | self = true 57 | security_group_id = aws_security_group.self.id 58 | } 59 | 60 | resource "aws_security_group_rule" "self_k3s_server" { 61 | type = "ingress" 62 | from_port = 6443 63 | to_port = 6443 64 | protocol = "TCP" 65 | cidr_blocks = local.private_subnets_cidr_blocks 66 | security_group_id = aws_security_group.self.id 67 | } 68 | 69 | resource "aws_security_group" "database" { 70 | name = "${local.name}-database" 71 | vpc_id = data.aws_vpc.default.id 72 | } 73 | 74 | resource "aws_security_group_rule" "database_self" { 75 | type = "ingress" 76 | from_port = 5432 77 | to_port = 5432 78 | protocol = "TCP" 79 | self = true 80 | security_group_id = aws_security_group.database.id 81 | } 82 | 83 | resource "aws_security_group_rule" "database_egress_all" { 84 | type = "egress" 85 | from_port = 0 86 | to_port = 0 87 | protocol = "-1" 88 | cidr_blocks = ["0.0.0.0/0"] 89 | security_group_id = aws_security_group.database.id 90 | } 91 | 92 | ############################# 93 | ### Create Nodes 94 | ############################# 95 | resource "aws_launch_template" "k3s_server" { 96 | name_prefix = "${local.name}-server" 97 | image_id = local.server_image_id 98 | instance_type = local.server_instance_type 99 | user_data = data.template_cloudinit_config.k3s_server.rendered 100 | 101 | block_device_mappings { 102 | device_name = "/dev/sda1" 103 | 104 | ebs { 105 | encrypted = true 106 | volume_type = "gp2" 107 | volume_size = "50" 108 | } 109 | } 110 | 111 | network_interfaces { 112 | delete_on_termination = true 113 | security_groups = concat([aws_security_group.self.id, aws_security_group.database.id], var.extra_server_security_groups) 114 | } 115 | 116 | tags = { 117 | Name = "${local.name}-server" 118 | } 119 | 120 | tag_specifications { 121 | resource_type = "instance" 122 | 123 | tags = { 124 | Name = "${local.name}-server" 125 | } 126 | } 127 | } 128 | 129 | resource "aws_launch_template" "k3s_agent" { 130 | name_prefix = "${local.name}-agent" 131 | image_id = local.agent_image_id 132 | instance_type = local.agent_instance_type 133 | user_data = data.template_cloudinit_config.k3s_agent.rendered 134 | 135 | block_device_mappings { 136 | device_name = "/dev/sda1" 137 | 138 | ebs { 139 | encrypted = true 140 | volume_type = "gp2" 141 | volume_size = "50" 142 | } 143 | } 144 | 145 | network_interfaces { 146 | delete_on_termination = true 147 | security_groups = concat([aws_security_group.ingress.id, aws_security_group.self.id], var.extra_agent_security_groups) 148 | } 149 | 150 | tags = { 151 | Name = "${local.name}-agent" 152 | } 153 | 154 | tag_specifications { 155 | resource_type = "instance" 156 | 157 | tags = { 158 | Name = "${local.name}-agent" 159 | } 160 | } 161 | } 162 | 163 | resource "aws_autoscaling_group" "k3s_server" { 164 | name_prefix = "${local.name}-server" 165 | desired_capacity = local.server_node_count 166 | max_size = local.server_node_count 167 | min_size = local.server_node_count 168 | vpc_zone_identifier = local.private_subnets 169 | 170 | target_group_arns = [ 171 | aws_lb_target_group.server-6443.arn 172 | ] 173 | 174 | launch_template { 175 | id = aws_launch_template.k3s_server.id 176 | version = "$Latest" 177 | } 178 | 179 | depends_on = [aws_rds_cluster_instance.k3s] 180 | } 181 | 182 | resource "aws_autoscaling_group" "k3s_agent" { 183 | name_prefix = "${local.name}-agent" 184 | desired_capacity = local.agent_node_count 185 | max_size = local.agent_node_count 186 | min_size = local.agent_node_count 187 | vpc_zone_identifier = local.private_subnets 188 | 189 | target_group_arns = [ 190 | aws_lb_target_group.agent-80.0.arn, 191 | aws_lb_target_group.agent-443.0.arn 192 | ] 193 | 194 | launch_template { 195 | id = aws_launch_template.k3s_agent.id 196 | version = "$Latest" 197 | } 198 | } 199 | 200 | ############################# 201 | ### Create Database 202 | ############################# 203 | resource "aws_db_subnet_group" "private" { 204 | count = local.deploy_rds 205 | name_prefix = "${local.name}-private" 206 | subnet_ids = local.private_subnets 207 | } 208 | 209 | resource "aws_rds_cluster_parameter_group" "k3s" { 210 | count = local.deploy_rds 211 | name_prefix = "${local.name}-" 212 | description = "Force SSL for aurora-postgresql10.7" 213 | family = "aurora-postgresql10" 214 | 215 | parameter { 216 | name = "rds.force_ssl" 217 | value = "1" 218 | apply_method = "pending-reboot" 219 | } 220 | } 221 | 222 | resource "aws_rds_cluster" "k3s" { 223 | count = local.deploy_rds 224 | cluster_identifier_prefix = "${local.name}-" 225 | engine = "aurora-postgresql" 226 | engine_mode = "provisioned" 227 | engine_version = "10.7" 228 | db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.k3s.0.name 229 | availability_zones = local.aws_azs 230 | database_name = local.db_name 231 | master_username = local.db_user 232 | master_password = local.db_pass 233 | preferred_maintenance_window = "fri:11:21-fri:11:51" 234 | db_subnet_group_name = aws_db_subnet_group.private.0.id 235 | vpc_security_group_ids = [aws_security_group.database.id] 236 | storage_encrypted = true 237 | 238 | preferred_backup_window = "11:52-19:52" 239 | backup_retention_period = 30 240 | copy_tags_to_snapshot = true 241 | deletion_protection = false 242 | skip_final_snapshot = local.skip_final_snapshot ? true : false 243 | final_snapshot_identifier = local.skip_final_snapshot ? null : "${local.name}-final-snapshot" 244 | } 245 | 246 | resource "aws_rds_cluster_instance" "k3s" { 247 | count = local.db_node_count 248 | identifier_prefix = "${local.name}-${count.index}" 249 | cluster_identifier = aws_rds_cluster.k3s.0.id 250 | engine = "aurora-postgresql" 251 | instance_class = local.db_instance_type 252 | db_subnet_group_name = aws_db_subnet_group.private.0.id 253 | } 254 | 255 | ############################# 256 | ### Create Public Rancher DNS 257 | ############################# 258 | resource "aws_route53_record" "rancher" { 259 | count = local.install_rancher ? local.create_external_nlb : 0 260 | zone_id = data.aws_route53_zone.dns_zone.zone_id 261 | name = "${local.name}.${local.domain}" 262 | type = "CNAME" 263 | ttl = 30 264 | records = [aws_lb.lb.0.dns_name] 265 | provider = aws.r53 266 | } 267 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | --------------------------------------------------------------------------------