├── .gitignore ├── README.md ├── gensubnets ├── Dockerfile ├── Makefile └── main.go ├── main.tf ├── modules ├── subnet-pair │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── vpc │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── outputs.tf ├── override.tf ├── update-zone.json └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | /.terraform 2 | *.tfstate 3 | *.tfstate.backup 4 | gensubnets/gensubnets 5 | /data 6 | terraform.tfvars 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-aws-vpc-kops-terraform 2 | 3 | Example code for 4 | the 5 | [Deploy Kubernetes in an Existing AWS VPC with Kops and Terraform](https://ryaneschinger.com/blog/kubernetes-aws-vpc-kops-terraform/) blog 6 | post. 7 | 8 | ## tldr 9 | 10 | ```bash 11 | terraform apply -var name=yourdomain.com 12 | 13 | export NAME=$(terraform output cluster_name) 14 | export KOPS_STATE_STORE=$(terraform output state_store) 15 | export ZONES=$(terraform output -json availability_zones | jq -r '.value|join(",")') 16 | 17 | kops create cluster \ 18 | --master-zones $ZONES \ 19 | --zones $ZONES \ 20 | --topology private \ 21 | --dns-zone $(terraform output public_zone_id) \ 22 | --networking calico \ 23 | --vpc $(terraform output vpc_id) \ 24 | --target=terraform \ 25 | --out=. \ 26 | ${NAME} 27 | 28 | terraform output -json | docker run --rm -i ryane/gensubnets:0.1 | pbcopy 29 | 30 | kops edit cluster ${NAME} 31 | 32 | # replace *subnets* section with your paste buffer (be careful to indent properly) 33 | # save and quit editor 34 | 35 | kops update cluster \ 36 | --out=. \ 37 | --target=terraform \ 38 | ${NAME} 39 | 40 | terraform apply -var name=yourdomain.com 41 | ``` 42 | 43 | ## using a subdomain 44 | 45 | If you want all of your dns records to live under a subdomain in its own hosted 46 | zone, you need to setup route delegation to the new zone. After running 47 | `terraform apply -var name=k8s.yourdomain.com`, you can run the following 48 | commands to setup the delegation: 49 | 50 | ```bash 51 | cat update-zone.json \ 52 | | jq ".Changes[].ResourceRecordSet.Name=\"$(terraform output name).\"" \ 53 | | jq ".Changes[].ResourceRecordSet.ResourceRecords=$(terraform output -json name_servers | jq '.value|[{"Value": .[]}]')" \ 54 | > update-zone.json 55 | 56 | aws --profile=default route53 change-resource-record-sets \ 57 | --hosted-zone-id $(aws --profile=default route53 list-hosted-zones | jq -r '.HostedZones[] | select(.Name=="yourdomain.com.") | .Id' | sed 's/\/hostedzone\///') \ 58 | --change-batch file://update-zone.json 59 | ``` 60 | 61 | Wait until your changes propagate before continuing. You are good to go when 62 | 63 | ```bash 64 | host -a k8s.yourdomain.com 65 | ``` 66 | 67 | returns the correct NS records. 68 | -------------------------------------------------------------------------------- /gensubnets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | ADD main / 3 | CMD ["/main"] 4 | -------------------------------------------------------------------------------- /gensubnets/Makefile: -------------------------------------------------------------------------------- 1 | NAME = $(shell awk -F\" '/^\t+name / { print $$2 }' ./main.go) 2 | VERSION = $(shell awk -F\" '/^\t+version / { print $$2 }' ./main.go) 3 | REPO = ryane 4 | 5 | build: $(shell find . -name '*.go') 6 | go build 7 | 8 | docker: $(shell find . -name '*.go') 9 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main . 10 | docker build --force-rm --rm -t ${REPO}/${NAME}:${VERSION} . 11 | rm -f main 12 | 13 | push: docker 14 | docker push ${REPO}/${NAME}:${VERSION} 15 | -------------------------------------------------------------------------------- /gensubnets/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | 11 | "github.com/ghodss/yaml" 12 | ) 13 | 14 | const ( 15 | name = "gensubnets" 16 | version = "0.1" 17 | ) 18 | 19 | type terraformOutput struct { 20 | AvailabilityZones map[string]interface{} `json:"availability_zones"` 21 | PrivateSubnetIDs map[string]interface{} `json:"private_subnet_ids"` 22 | PublicSubnetIDs map[string]interface{} `json:"public_subnet_ids"` 23 | NATGatewayIDs map[string]interface{} `json:"nat_gateway_ids"` 24 | } 25 | 26 | type subnetSpec struct { 27 | Name string `json:"name,omitempty"` 28 | Zone string `json:"zone,omitempty"` 29 | CIDR string `json:"cidr,omitempty"` 30 | ProviderID string `json:"id,omitempty"` 31 | Egress string `json:"egress,omitempty"` 32 | Type string `json:"type,omitempty"` 33 | } 34 | 35 | type subnetSpecs struct { 36 | Subnets []subnetSpec `json:"subnets"` 37 | } 38 | 39 | func main() { 40 | flag.Parse() 41 | 42 | var tfJSON []byte 43 | var err error 44 | if flag.NArg() >= 1 { 45 | tfJSON, err = ioutil.ReadFile(flag.Arg(0)) 46 | } else { 47 | tfJSON, err = ioutil.ReadAll(os.Stdin) 48 | } 49 | 50 | if err != nil { 51 | log.Fatal(err) 52 | } 53 | 54 | var tfOut terraformOutput 55 | err = json.Unmarshal(tfJSON, &tfOut) 56 | if err != nil { 57 | log.Fatal(err) 58 | } 59 | 60 | azs := getValues(tfOut.AvailabilityZones) 61 | azCount := len(azs) 62 | privateSubnets := getValues(tfOut.PrivateSubnetIDs) 63 | publicSubnets := getValues(tfOut.PublicSubnetIDs) 64 | natGateways := getValues(tfOut.NATGatewayIDs) 65 | 66 | subnets := make([]subnetSpec, azCount*2) 67 | for i, subnetID := range privateSubnets { 68 | subnets[i] = subnetSpec{ 69 | ProviderID: subnetID, 70 | Egress: natGateways[i], 71 | Name: azs[i], 72 | Type: "Private", 73 | Zone: azs[i], 74 | } 75 | } 76 | 77 | for i, subnetID := range publicSubnets { 78 | idx := i + azCount 79 | subnets[idx] = subnetSpec{ 80 | ProviderID: subnetID, 81 | Name: "utility-" + azs[i], 82 | Type: "Utility", 83 | Zone: azs[i], 84 | } 85 | } 86 | 87 | subnetSpecs := subnetSpecs{Subnets: subnets} 88 | data, err := yaml.Marshal(subnetSpecs) 89 | if err != nil { 90 | log.Fatal(err) 91 | } 92 | fmt.Printf("%+v\n", string(data)) 93 | } 94 | 95 | func getValues(m map[string]interface{}) []string { 96 | if rawv, ok := m["value"]; ok { 97 | if slice, ok := rawv.([]interface{}); ok { 98 | vals := make([]string, len(slice)) 99 | for i, val := range slice { 100 | vals[i] = val.(string) 101 | } 102 | return vals 103 | } 104 | } 105 | return []string{} 106 | } 107 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | module "vpc" { 2 | source = "./modules/vpc" 3 | name = "${var.name}" 4 | env = "${var.env}" 5 | vpc_cidr = "${var.vpc_cidr}" 6 | 7 | tags { 8 | Infra = "${var.name}" 9 | Environment = "${var.env}" 10 | Terraformed = "true" 11 | KubernetesCluster = "${var.env}.${var.name}" 12 | } 13 | } 14 | 15 | module "subnet_pair" { 16 | source = "./modules/subnet-pair" 17 | name = "${var.name}" 18 | env = "${var.env}" 19 | vpc_id = "${module.vpc.vpc_id}" 20 | vpc_cidr = "${module.vpc.cidr_block}" 21 | internet_gateway_id = "${module.vpc.internet_gateway_id}" 22 | availability_zones = "${var.azs}" 23 | 24 | tags { 25 | Infra = "${var.name}" 26 | Environment = "${var.env}" 27 | Terraformed = "true" 28 | KubernetesCluster = "${var.env}.${var.name}" 29 | } 30 | } 31 | 32 | resource "aws_route53_zone" "public" { 33 | name = "${var.name}" 34 | force_destroy = true 35 | 36 | tags { 37 | Name = "${var.name}-${var.env}-zone-public" 38 | Infra = "${var.name}" 39 | Environment = "${var.env}" 40 | Terraformed = "true" 41 | } 42 | } 43 | 44 | resource "aws_s3_bucket" "state_store" { 45 | bucket = "${var.name}-state" 46 | acl = "private" 47 | force_destroy = true 48 | 49 | versioning { 50 | enabled = true 51 | } 52 | 53 | tags { 54 | Name = "${var.name}-${var.env}-state-store" 55 | Infra = "${var.name}" 56 | Environment = "${var.env}" 57 | Terraformed = "true" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /modules/subnet-pair/main.tf: -------------------------------------------------------------------------------- 1 | data "aws_availability_zone" "az" { 2 | count = "${length(var.availability_zones)}" 3 | name = "${var.availability_zones[count.index]}" 4 | } 5 | 6 | resource "aws_subnet" "public" { 7 | count = "${length(var.availability_zones)}" 8 | vpc_id = "${var.vpc_id}" 9 | cidr_block = "${cidrsubnet(var.vpc_cidr, var.newbits, var.az_number[data.aws_availability_zone.az.*.name_suffix[count.index]] + var.public_netnum_offset)}" 10 | availability_zone = "${var.availability_zones[count.index]}" 11 | map_public_ip_on_launch = true 12 | tags = "${merge(map("Name", "${var.name}-${var.env}-sn-public-${data.aws_availability_zone.az.*.name_suffix[count.index]}"), var.tags)}" 13 | } 14 | 15 | resource "aws_route_table" "public" { 16 | vpc_id = "${var.vpc_id}" 17 | tags = "${merge(map("Name", "${var.name}-${var.env}-rt-public"), var.tags)}" 18 | } 19 | 20 | resource "aws_route" "internet_route" { 21 | route_table_id = "${aws_route_table.public.id}" 22 | destination_cidr_block = "0.0.0.0/0" 23 | gateway_id = "${var.internet_gateway_id}" 24 | 25 | lifecycle { 26 | create_before_destroy = true 27 | } 28 | } 29 | 30 | resource "aws_route_table_association" "public" { 31 | count = "${length(var.availability_zones)}" 32 | subnet_id = "${aws_subnet.public.*.id[count.index]}" 33 | route_table_id = "${aws_route_table.public.id}" 34 | 35 | lifecycle { 36 | ignore_changes = ["subnet_id", "route_table_id"] 37 | create_before_destroy = true 38 | } 39 | } 40 | 41 | resource "aws_eip" "nat_eip" { 42 | count = "${length(var.availability_zones)}" 43 | vpc = true 44 | } 45 | 46 | resource "aws_nat_gateway" "nat_gw" { 47 | count = "${length(var.availability_zones)}" 48 | allocation_id = "${aws_eip.nat_eip.*.id[count.index]}" 49 | subnet_id = "${aws_subnet.public.*.id[count.index]}" 50 | 51 | lifecycle { 52 | create_before_destroy = true 53 | ignore_changes = ["subnet_id"] 54 | } 55 | } 56 | 57 | resource "aws_subnet" "private" { 58 | count = "${length(var.availability_zones)}" 59 | vpc_id = "${var.vpc_id}" 60 | cidr_block = "${cidrsubnet(var.vpc_cidr, var.newbits, var.az_number[data.aws_availability_zone.az.*.name_suffix[count.index]] + var.private_netnum_offset)}" 61 | availability_zone = "${var.availability_zones[count.index]}" 62 | map_public_ip_on_launch = false 63 | tags = "${merge(map("Name", "${var.name}-${var.env}-sn-private-${data.aws_availability_zone.az.*.name_suffix[count.index]}"), var.tags)}" 64 | depends_on = ["aws_nat_gateway.nat_gw"] 65 | } 66 | 67 | resource "aws_route_table" "private" { 68 | count = "${length(var.availability_zones)}" 69 | vpc_id = "${var.vpc_id}" 70 | tags = "${merge(map("Name", "${var.name}-${var.env}-rt-private-${data.aws_availability_zone.az.*.name_suffix[count.index]}"), var.tags)}" 71 | } 72 | 73 | resource "aws_route" "nat_route" { 74 | count = "${length(var.availability_zones)}" 75 | route_table_id = "${aws_route_table.private.*.id[count.index]}" 76 | destination_cidr_block = "0.0.0.0/0" 77 | nat_gateway_id = "${aws_nat_gateway.nat_gw.*.id[count.index]}" 78 | 79 | lifecycle { 80 | create_before_destroy = true 81 | ignore_changes = ["route_table_id", "nat_gateway_id"] 82 | } 83 | 84 | depends_on = ["aws_nat_gateway.nat_gw"] 85 | } 86 | 87 | resource "aws_route_table_association" "private" { 88 | count = "${length(var.availability_zones)}" 89 | subnet_id = "${aws_subnet.private.*.id[count.index]}" 90 | route_table_id = "${aws_route_table.private.*.id[count.index]}" 91 | 92 | lifecycle { 93 | ignore_changes = ["subnet_id"] 94 | create_before_destroy = true 95 | } 96 | } 97 | 98 | resource "aws_security_group" "default" { 99 | name = "internal" 100 | description = "Default security group that allows inbound and outbound traffic from all instances in the VPC" 101 | vpc_id = "${var.vpc_id}" 102 | tags = "${merge(map("Name", "${var.name}-${var.env}-sg-default"), var.tags)}" 103 | } 104 | 105 | resource "aws_security_group_rule" "internal_ingress" { 106 | type = "ingress" 107 | from_port = "0" 108 | to_port = "0" 109 | protocol = "-1" 110 | self = true 111 | security_group_id = "${aws_security_group.default.id}" 112 | 113 | lifecycle { 114 | create_before_destroy = true 115 | } 116 | } 117 | 118 | resource "aws_security_group_rule" "internal_egress" { 119 | type = "egress" 120 | from_port = "0" 121 | to_port = "0" 122 | protocol = "-1" 123 | self = true 124 | security_group_id = "${aws_security_group.default.id}" 125 | 126 | lifecycle { 127 | create_before_destroy = true 128 | } 129 | } 130 | 131 | resource "aws_security_group" "nat" { 132 | name = "nat" 133 | description = "security group that allows all inbound and outbound traffic. should only be applied to instances in a private subnet" 134 | vpc_id = "${var.vpc_id}" 135 | tags = "${merge(map("Name", "${var.name}-${var.env}-sg-nat"), var.tags)}" 136 | depends_on = ["aws_nat_gateway.nat_gw"] 137 | } 138 | 139 | resource "aws_security_group_rule" "nat_ingress" { 140 | type = "ingress" 141 | from_port = "0" 142 | to_port = "0" 143 | protocol = "-1" 144 | cidr_blocks = ["0.0.0.0/0"] 145 | security_group_id = "${aws_security_group.nat.id}" 146 | 147 | lifecycle { 148 | create_before_destroy = true 149 | } 150 | } 151 | 152 | resource "aws_security_group_rule" "nat_egress" { 153 | type = "egress" 154 | from_port = "0" 155 | to_port = "0" 156 | protocol = "-1" 157 | cidr_blocks = ["0.0.0.0/0"] 158 | security_group_id = "${aws_security_group.nat.id}" 159 | 160 | lifecycle { 161 | create_before_destroy = true 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /modules/subnet-pair/outputs.tf: -------------------------------------------------------------------------------- 1 | output "public_subnet_ids" { 2 | value = ["${aws_subnet.public.*.id}"] 3 | } 4 | 5 | output "private_subnet_ids" { 6 | value = ["${aws_subnet.private.*.id}"] 7 | } 8 | 9 | output "default_security_group_id" { 10 | value = "${aws_security_group.default.id}" 11 | } 12 | 13 | output "nat_gateway_ids" { 14 | value = ["${aws_nat_gateway.nat_gw.*.id}"] 15 | } 16 | 17 | output "nat_security_group_id" { 18 | value = "${aws_security_group.nat.id}" 19 | } 20 | 21 | output "availability_zones" { 22 | value = "${var.availability_zones}" 23 | } 24 | -------------------------------------------------------------------------------- /modules/subnet-pair/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" {} 2 | 3 | variable "env" {} 4 | 5 | variable "vpc_id" {} 6 | 7 | variable "vpc_cidr" {} 8 | 9 | variable "availability_zones" { 10 | type = "list" 11 | } 12 | 13 | variable "internet_gateway_id" {} 14 | 15 | variable "newbits" { 16 | default = 8 17 | description = "number of bits to add to the vpc cidr when building subnets" 18 | } 19 | 20 | variable "az_number" { 21 | default = { 22 | a = 1 23 | b = 2 24 | c = 3 25 | d = 4 26 | e = 5 27 | f = 6 28 | } 29 | } 30 | 31 | variable "public_netnum_offset" { 32 | default = 0 33 | } 34 | 35 | variable "private_netnum_offset" { 36 | default = 100 37 | } 38 | 39 | variable "tags" { 40 | type = "map" 41 | default = {} 42 | } 43 | -------------------------------------------------------------------------------- /modules/vpc/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_vpc" "default" { 2 | cidr_block = "${var.vpc_cidr}" 3 | enable_dns_hostnames = true 4 | tags = "${merge(map("Name", "${var.name}-${var.env}-vpc"), var.tags)}" 5 | } 6 | 7 | resource "aws_internet_gateway" "default" { 8 | vpc_id = "${aws_vpc.default.id}" 9 | tags = "${merge(map("Name", "${var.name}-${var.env}-ig"), var.tags)}" 10 | } 11 | -------------------------------------------------------------------------------- /modules/vpc/outputs.tf: -------------------------------------------------------------------------------- 1 | output "vpc_id" { 2 | value = "${aws_vpc.default.id}" 3 | } 4 | 5 | output "cidr_block" { 6 | value = "${aws_vpc.default.cidr_block}" 7 | } 8 | 9 | output "internet_gateway_id" { 10 | value = "${aws_internet_gateway.default.id}" 11 | } 12 | -------------------------------------------------------------------------------- /modules/vpc/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" {} 2 | 3 | variable "env" {} 4 | 5 | variable "vpc_cidr" { 6 | default = "10.0.0.0/16" 7 | } 8 | 9 | variable "tags" { 10 | type = "map" 11 | default = {} 12 | } 13 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | output "name" { 2 | value = "${var.name}" 3 | } 4 | 5 | output "cluster_name" { 6 | value = "${var.env}.${var.name}" 7 | } 8 | 9 | output "vpc_id" { 10 | value = "${module.vpc.vpc_id}" 11 | } 12 | 13 | output "name_servers" { 14 | value = "${aws_route53_zone.public.name_servers}" 15 | } 16 | 17 | output "public_zone_id" { 18 | value = "${aws_route53_zone.public.zone_id}" 19 | } 20 | 21 | output "state_store" { 22 | value = "s3://${aws_s3_bucket.state_store.id}" 23 | } 24 | 25 | output "public_subnet_ids" { 26 | value = "${module.subnet_pair.public_subnet_ids}" 27 | } 28 | 29 | output "private_subnet_ids" { 30 | value = "${module.subnet_pair.private_subnet_ids}" 31 | } 32 | 33 | output "nat_gateway_ids" { 34 | value = "${module.subnet_pair.nat_gateway_ids}" 35 | } 36 | 37 | output "availability_zones" { 38 | value = "${module.subnet_pair.availability_zones}" 39 | } 40 | -------------------------------------------------------------------------------- /override.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "${var.region}" 3 | } 4 | -------------------------------------------------------------------------------- /update-zone.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "Create a subdomain NS record in the parent domain", 3 | "Changes": [ 4 | { 5 | "Action": "UPSERT", 6 | "ResourceRecordSet": { 7 | "Name": "", 8 | "Type": "NS", 9 | "TTL": 300, 10 | "ResourceRecords": [] 11 | } 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | default = "example.com" 3 | } 4 | 5 | variable "region" { 6 | default = "us-east-1" 7 | } 8 | 9 | variable "azs" { 10 | default = ["us-east-1a", "us-east-1c", "us-east-1d"] 11 | type = "list" 12 | } 13 | 14 | variable "env" { 15 | default = "staging" 16 | } 17 | 18 | variable "vpc_cidr" { 19 | default = "10.20.0.0/16" 20 | } 21 | --------------------------------------------------------------------------------