├── .gitignore ├── README.md ├── kubernetes-cluster ├── cluster-template.yaml └── regen-cluster.sh └── terraform ├── main.tf ├── outputs.tf ├── s3_buckets.tf ├── security_groups.tf └── vpc.tf /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | ./*.tfstate 3 | .terraform 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deploying Kubernetes clusters with kops and Terraform 2 | 3 | Deploy a Kubernetes cluster using Terraformed AWS resources and kops. Used for the blog post https://medium.com/bench-engineering/deploying-kubernetes-clusters-with-kops-and-terraform-832b89250e8e 4 | 5 | ## Requirements 6 | 7 | * jq 8 | * kops 9 | * kubectl 10 | * terraform 11 | 12 | ## Usage 13 | 14 | Edit `terraform/main.tf` with your local variables (details in the blog post above) 15 | 16 | From the `terraform` directory run: 17 | 18 | terraform init 19 | terraform plan 20 | terraform apply 21 | 22 | Then from the `kubernetes-cluster` dir run: 23 | 24 | ./regen-cluster.sh 25 | terraform plan 26 | terraform apply 27 | -------------------------------------------------------------------------------- /kubernetes-cluster/cluster-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kops/v1alpha2 2 | kind: Cluster 3 | metadata: 4 | name: {{.kubernetes_cluster_name.value}} 5 | spec: 6 | api: 7 | loadBalancer: 8 | type: Public 9 | additionalSecurityGroups: ["{{.common_http_sg_id.value}}"] 10 | authorization: 11 | rbac: {} 12 | channel: stable 13 | cloudProvider: aws 14 | configBase: s3://{{.kops_s3_bucket.value}}/{{.kubernetes_cluster_name.value}} 15 | # Create one etcd member per AZ 16 | etcdClusters: 17 | - etcdMembers: 18 | {{range $i, $az := .availability_zones.value}} 19 | - instanceGroup: master-{{.}} 20 | name: {{. | replace $.region.value "" }} {{/* converts eu-west-1a to a */}} 21 | {{end}} 22 | name: main 23 | - etcdMembers: 24 | {{range $i, $az := .availability_zones.value}} 25 | - instanceGroup: master-{{.}} 26 | name: {{. | replace $.region.value "" }} {{/* converts eu-west-1a to a */}} 27 | {{end}} 28 | name: events 29 | iam: 30 | allowContainerRegistry: true 31 | legacy: false 32 | kubernetesVersion: 1.10.6 33 | masterPublicName: api.{{.kubernetes_cluster_name.value}} 34 | networkCIDR: {{.vpc_cidr_block.value}} 35 | networkID: {{.vpc_id.value}} 36 | networking: 37 | canal: {} 38 | nonMasqueradeCIDR: 100.64.0.0/10 39 | subnets: 40 | # Public (utility) subnets, one per AZ 41 | {{range $i, $id := .public_subnet_ids.value}} 42 | - id: {{.}} 43 | name: utility-{{index $.availability_zones.value $i}} 44 | type: Utility 45 | zone: {{index $.availability_zones.value $i}} 46 | {{end}} 47 | # Private subnets, one per AZ 48 | {{range $i, $id := .private_subnet_ids.value}} 49 | - id: {{.}} 50 | name: {{index $.availability_zones.value $i}} 51 | type: Private 52 | zone: {{index $.availability_zones.value $i}} 53 | egress: {{index $.nat_gateway_ids.value $i}} 54 | {{end}} 55 | topology: 56 | dns: 57 | type: Public 58 | masters: private 59 | nodes: private 60 | --- 61 | 62 | # Create one master per AZ 63 | {{range .availability_zones.value}} 64 | apiVersion: kops/v1alpha2 65 | kind: InstanceGroup 66 | metadata: 67 | labels: 68 | kops.k8s.io/cluster: {{$.kubernetes_cluster_name.value}} 69 | name: master-{{.}} 70 | spec: 71 | image: kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-08-17 72 | kubernetesVersion: 1.10.6 73 | machineType: t2.medium 74 | maxSize: 1 75 | minSize: 1 76 | role: Master 77 | subnets: 78 | - {{.}} 79 | --- 80 | {{end}} 81 | 82 | apiVersion: kops/v1alpha2 83 | kind: InstanceGroup 84 | metadata: 85 | labels: 86 | kops.k8s.io/cluster: {{.kubernetes_cluster_name.value}} 87 | name: nodes 88 | spec: 89 | image: kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-08-17 90 | kubernetesVersion: 1.10.6 91 | machineType: t2.small 92 | maxSize: 2 93 | minSize: 1 94 | role: Node 95 | subnets: 96 | {{range .availability_zones.value}} 97 | - {{.}} 98 | {{end}} 99 | -------------------------------------------------------------------------------- /kubernetes-cluster/regen-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e -o pipefail 4 | 5 | TF_OUTPUT=$(cd ../terraform && terraform output -json) 6 | CLUSTER_NAME="$(echo ${TF_OUTPUT} | jq -r .kubernetes_cluster_name.value)" 7 | STATE="s3://$(echo ${TF_OUTPUT} | jq -r .kops_s3_bucket.value)" 8 | 9 | kops toolbox template --name ${CLUSTER_NAME} --values <( echo ${TF_OUTPUT}) --template cluster-template.yaml --format-yaml > cluster.yaml 10 | 11 | kops replace -f cluster.yaml --state ${STATE} --name ${CLUSTER_NAME} --force 12 | 13 | kops update cluster --target terraform --state ${STATE} --name ${CLUSTER_NAME} --out . 14 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "eu-west-2" 3 | } 4 | 5 | terraform { 6 | backend "s3" { 7 | bucket = "tf-state-blog" 8 | key = "dev/terraform" 9 | region = "eu-west-2" 10 | } 11 | } 12 | 13 | locals { 14 | azs = ["eu-west-2a", "eu-west-2b", "eu-west-2c"] 15 | environment = "dev" 16 | kops_state_bucket_name = "${local.environment}-kops-state" 17 | // Needs to be a FQDN 18 | kubernetes_cluster_name = "k8s-dev0.domain.com" 19 | ingress_ips = ["10.0.0.100/32", "10.0.0.101/32"] 20 | vpc_name = "${local.environment}-vpc" 21 | 22 | tags = { 23 | environment = "${local.environment}" 24 | terraform = true 25 | } 26 | } 27 | 28 | data "aws_region" "current" {} 29 | -------------------------------------------------------------------------------- /terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "region" { 2 | value = "${data.aws_region.current.name}" 3 | } 4 | 5 | output "vpc_id" { 6 | value = "${module.dev_vpc.vpc_id}" 7 | } 8 | 9 | output "vpc_name" { 10 | value = "${local.vpc_name}" 11 | } 12 | 13 | output "vpc_cidr_block" { 14 | value = "${module.dev_vpc.vpc_cidr_block}" 15 | } 16 | 17 | // Public Subnets 18 | 19 | output "public_subnet_ids" { 20 | value = ["${module.dev_vpc.public_subnets}"] 21 | } 22 | 23 | output "public_route_table_ids" { 24 | value = ["${module.dev_vpc.public_route_table_ids}"] 25 | } 26 | 27 | // Private Subnets 28 | 29 | output "private_subnet_ids" { 30 | value = ["${module.dev_vpc.private_subnets}"] 31 | } 32 | 33 | output "private_route_table_ids" { 34 | value = ["${module.dev_vpc.private_route_table_ids}"] 35 | } 36 | 37 | output "default_security_group_id" { 38 | value = "${module.dev_vpc.default_security_group_id}" 39 | } 40 | 41 | output "nat_gateway_ids" { 42 | value = "${module.dev_vpc.natgw_ids}" 43 | } 44 | 45 | output "availability_zones" { 46 | value = ["${local.azs}"] 47 | } 48 | 49 | output "common_http_sg_id" { 50 | value = "${aws_security_group.k8s_common_http.id}" 51 | } 52 | 53 | // Needed for kops 54 | 55 | output "kops_s3_bucket" { 56 | value = "${aws_s3_bucket.kops_state.bucket}" 57 | } 58 | 59 | output "kubernetes_cluster_name" { 60 | value = "${local.kubernetes_cluster_name}" 61 | } 62 | -------------------------------------------------------------------------------- /terraform/s3_buckets.tf: -------------------------------------------------------------------------------- 1 | // S3 bucket to store kops state. 2 | resource "aws_s3_bucket" "kops_state" { 3 | bucket = "${local.kops_state_bucket_name}" 4 | acl = "private" 5 | force_destroy = true 6 | tags = "${merge(local.tags)}" 7 | } 8 | -------------------------------------------------------------------------------- /terraform/security_groups.tf: -------------------------------------------------------------------------------- 1 | // Used to allow web access to the k8s API ELB 2 | resource "aws_security_group" "k8s_common_http" { 3 | name = "${local.environment}_k8s_common_http" 4 | vpc_id = "${module.dev_vpc.vpc_id}" 5 | tags = "${merge(local.tags)}" 6 | 7 | ingress { 8 | from_port = 80 9 | protocol = "tcp" 10 | to_port = 80 11 | cidr_blocks = ["${local.ingress_ips}"] 12 | } 13 | 14 | ingress { 15 | from_port = 443 16 | protocol = "tcp" 17 | to_port = 443 18 | cidr_blocks = ["${local.ingress_ips}"] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /terraform/vpc.tf: -------------------------------------------------------------------------------- 1 | module "dev_vpc" { 2 | source = "terraform-aws-modules/vpc/aws" 3 | version = "1.46.0" 4 | name = "${local.vpc_name}" 5 | cidr = "10.0.0.0/16" 6 | azs = ["${local.azs}"] 7 | private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] 8 | public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] 9 | enable_nat_gateway = true 10 | 11 | tags = { 12 | // This is so kops knows that the VPC resources can be used for k8s 13 | "kubernetes.io/cluster/${local.kubernetes_cluster_name}" = "shared" 14 | "terraform" = true 15 | "environment" = "${local.environment}" 16 | } 17 | 18 | // Tags required by k8s to launch services on the right subnets 19 | private_subnet_tags = { 20 | "kubernetes.io/role/internal-elb" = true 21 | } 22 | 23 | public_subnet_tags = { 24 | "kubernetes.io/role/elb" = true 25 | } 26 | } 27 | --------------------------------------------------------------------------------