├── .gitignore ├── LICENSE ├── README.md ├── docs └── k3s-on-gcp.png ├── k3s-agents ├── igm.tf ├── network.tf ├── templates │ └── agent.sh └── variables.tf ├── k3s-db ├── db.tf ├── network.tf ├── outputs.tf └── variables.tf ├── k3s-servers ├── igm.tf ├── lb.tf ├── network.tf ├── outputs.tf ├── templates │ └── server.sh └── variables.tf ├── main.tf ├── terraform.example.tfvars └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | *.tfstate 2 | *.tfstate.backup 3 | terraform.tfvars 4 | 5 | .terraform/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Johan Siebens 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multi Region k3s cluster on GCP 2 | 3 | 4 | A HA k3s cluster build with: 5 | 6 | - a [Cloud SQL](https://cloud.google.com/sql) instance of an external datastore 7 | - a [Managed Instance Group](https://cloud.google.com/compute/docs/instance-groups) of server nodes that will serve the Kubernetes API and run other control plane services 8 | - multiple [Managed Instance Groups](https://cloud.google.com/compute/docs/instance-groups) of agent nodes that will run our apps, spread across multiple regions 9 | - an [Internal TCP Load Balancer](https://cloud.google.com/load-balancing/docs/internal) in front of the server nodes to allow the agent nodes to register with the cluster 10 | - an [External TCP Load Balancer](https://cloud.google.com/load-balancing/docs/network) to expose to API server to allow interaction with the cluster using e.g. `kubectl` 11 | 12 | 13 | -------------------------------------------------------------------------------- /docs/k3s-on-gcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsiebens/k3s-on-gcp/0211fce81c70f8090144947a4d633a8dbf62c4ac/docs/k3s-on-gcp.png -------------------------------------------------------------------------------- /k3s-agents/igm.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "k3s-agent-startup-script" { 2 | template = file("${path.module}/templates/agent.sh") 3 | vars = { 4 | token = var.token 5 | server_address = var.server_address 6 | } 7 | } 8 | 9 | resource "google_compute_instance_template" "k3s-agent" { 10 | name_prefix = "k3s-agent-${var.name}-" 11 | machine_type = var.machine_type 12 | 13 | tags = ["k3s", "k3s-agent"] 14 | 15 | metadata_startup_script = data.template_file.k3s-agent-startup-script.rendered 16 | 17 | metadata = { 18 | block-project-ssh-keys = "TRUE" 19 | enable-oslogin = "TRUE" 20 | } 21 | 22 | disk { 23 | source_image = "debian-cloud/debian-10" 24 | auto_delete = true 25 | boot = true 26 | } 27 | 28 | network_interface { 29 | network = var.network 30 | subnetwork = google_compute_subnetwork.k3s-agents.self_link 31 | } 32 | 33 | shielded_instance_config { 34 | enable_secure_boot = true 35 | } 36 | 37 | service_account { 38 | email = var.service_account 39 | scopes = [ 40 | "https://www.googleapis.com/auth/cloud-platform", 41 | ] 42 | } 43 | 44 | lifecycle { 45 | create_before_destroy = true 46 | } 47 | } 48 | 49 | resource "google_compute_region_instance_group_manager" "k3s-agents" { 50 | name = "k3s-agents-${var.name}" 51 | 52 | base_instance_name = "k3s-agent-${var.name}" 53 | region = var.region 54 | 55 | version { 56 | instance_template = google_compute_instance_template.k3s-agent.id 57 | } 58 | 59 | target_size = var.target_size 60 | 61 | named_port { 62 | name = "http" 63 | port = 80 64 | } 65 | 66 | named_port { 67 | name = "https" 68 | port = 443 69 | } 70 | 71 | update_policy { 72 | type = "PROACTIVE" 73 | instance_redistribution_type = "PROACTIVE" 74 | minimal_action = "REPLACE" 75 | max_surge_fixed = 3 76 | } 77 | 78 | depends_on = [google_compute_router_nat.nat] 79 | } 80 | -------------------------------------------------------------------------------- /k3s-agents/network.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_subnetwork" "k3s-agents" { 2 | name = "k3s-agents-${var.name}" 3 | network = var.network 4 | region = var.region 5 | ip_cidr_range = var.cidr_range 6 | 7 | private_ip_google_access = true 8 | } 9 | 10 | resource "google_compute_router" "router" { 11 | name = "k3s-agents-${var.name}" 12 | region = var.region 13 | network = var.network 14 | } 15 | 16 | resource "google_compute_router_nat" "nat" { 17 | name = "k3s-agents-${var.name}" 18 | router = google_compute_router.router.name 19 | region = var.region 20 | nat_ip_allocate_option = "AUTO_ONLY" 21 | source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" 22 | subnetwork { 23 | name = google_compute_subnetwork.k3s-agents.id 24 | source_ip_ranges_to_nat = ["ALL_IP_RANGES"] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /k3s-agents/templates/agent.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | curl -sfL https://get.k3s.io | K3S_TOKEN="${token}" INSTALL_K3S_VERSION="v1.19.3+k3s3" K3S_URL="https://${server_address}:6443" sh -s - \ 4 | --node-label "svccontroller.k3s.cattle.io/enablelb=true" -------------------------------------------------------------------------------- /k3s-agents/variables.tf: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | } 4 | 5 | variable "name" { 6 | type = string 7 | } 8 | 9 | variable "network" { 10 | type = string 11 | } 12 | 13 | variable "region" { 14 | type = string 15 | } 16 | 17 | variable "cidr_range" { 18 | } 19 | 20 | variable "machine_type" { 21 | type = string 22 | } 23 | 24 | variable "target_size" { 25 | type = number 26 | default = 3 27 | } 28 | 29 | variable "token" { 30 | } 31 | 32 | variable "server_address" { 33 | } 34 | 35 | variable "service_account" { 36 | } 37 | -------------------------------------------------------------------------------- /k3s-db/db.tf: -------------------------------------------------------------------------------- 1 | resource "random_id" "k3s-db" { 2 | prefix = "k3s-db-" 3 | byte_length = 4 4 | } 5 | 6 | resource "google_sql_database_instance" "k3s-db" { 7 | name = random_id.k3s-db.hex 8 | region = var.region 9 | database_version = "POSTGRES_11" 10 | 11 | settings { 12 | tier = var.db_tier 13 | availability_type = "REGIONAL" 14 | disk_size = 50 15 | disk_type = "PD_SSD" 16 | disk_autoresize = true 17 | 18 | ip_configuration { 19 | ipv4_enabled = "false" 20 | private_network = var.network 21 | } 22 | 23 | backup_configuration { 24 | enabled = true 25 | start_time = "01:00" 26 | } 27 | 28 | maintenance_window { 29 | day = 6 30 | hour = 1 31 | } 32 | } 33 | 34 | depends_on = [google_service_networking_connection.k3s-private-vpc-connection] 35 | } 36 | 37 | resource "google_sql_database" "k3s-db" { 38 | name = "k3s" 39 | instance = google_sql_database_instance.k3s-db.name 40 | } 41 | 42 | resource "random_password" "k3s-db-pwd" { 43 | length = 16 44 | special = false 45 | } 46 | 47 | resource "google_sql_user" "k3s" { 48 | name = "k3s" 49 | instance = google_sql_database_instance.k3s-db.name 50 | password = random_password.k3s-db-pwd.result 51 | } 52 | -------------------------------------------------------------------------------- /k3s-db/network.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_global_address" "k3s-db" { 2 | name = "k3s-db" 3 | purpose = "VPC_PEERING" 4 | address_type = "INTERNAL" 5 | prefix_length = 16 6 | network = var.network 7 | } 8 | 9 | resource "google_service_networking_connection" "k3s-private-vpc-connection" { 10 | network = var.network 11 | service = "servicenetworking.googleapis.com" 12 | reserved_peering_ranges = [google_compute_global_address.k3s-db.name] 13 | } 14 | -------------------------------------------------------------------------------- /k3s-db/outputs.tf: -------------------------------------------------------------------------------- 1 | output "db_host" { 2 | value = google_sql_database_instance.k3s-db.private_ip_address 3 | } 4 | 5 | output "db_name" { 6 | value = google_sql_database.k3s-db.name 7 | } 8 | 9 | output "db_user" { 10 | value = google_sql_user.k3s.name 11 | } 12 | 13 | output "db_password" { 14 | value = google_sql_user.k3s.password 15 | } 16 | -------------------------------------------------------------------------------- /k3s-db/variables.tf: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | } 4 | 5 | variable "network" { 6 | type = string 7 | } 8 | 9 | variable "region" { 10 | type = string 11 | } 12 | 13 | variable "db_tier" { 14 | type = string 15 | } 16 | -------------------------------------------------------------------------------- /k3s-servers/igm.tf: -------------------------------------------------------------------------------- 1 | resource "random_string" "token" { 2 | length = 32 3 | special = false 4 | } 5 | 6 | data "template_file" "k3s-server-startup-script" { 7 | template = file("${path.module}/templates/server.sh") 8 | vars = { 9 | token = random_string.token.result 10 | internal_lb_ip_address = google_compute_address.k3s-api-server-internal.address 11 | external_lb_ip_address = google_compute_address.k3s-api-server-external.address 12 | db_host = var.db_host 13 | db_name = var.db_name 14 | db_user = var.db_user 15 | db_password = var.db_password 16 | } 17 | } 18 | 19 | resource "google_compute_instance_template" "k3s-server" { 20 | name_prefix = "k3s-server-" 21 | machine_type = var.machine_type 22 | 23 | tags = ["k3s", "k3s-server"] 24 | 25 | metadata_startup_script = data.template_file.k3s-server-startup-script.rendered 26 | 27 | metadata = { 28 | block-project-ssh-keys = "TRUE" 29 | enable-oslogin = "TRUE" 30 | } 31 | 32 | disk { 33 | source_image = "debian-cloud/debian-10" 34 | auto_delete = true 35 | boot = true 36 | } 37 | 38 | network_interface { 39 | network = var.network 40 | subnetwork = google_compute_subnetwork.k3s-servers.id 41 | } 42 | 43 | shielded_instance_config { 44 | enable_secure_boot = true 45 | } 46 | 47 | service_account { 48 | email = var.service_account 49 | scopes = [ 50 | "https://www.googleapis.com/auth/cloud-platform", 51 | ] 52 | } 53 | 54 | lifecycle { 55 | create_before_destroy = true 56 | } 57 | } 58 | 59 | resource "google_compute_region_instance_group_manager" "k3s-servers" { 60 | name = "k3s-servers" 61 | 62 | base_instance_name = "k3s-server" 63 | region = var.region 64 | 65 | version { 66 | instance_template = google_compute_instance_template.k3s-server.id 67 | } 68 | 69 | target_size = var.target_size 70 | 71 | named_port { 72 | name = "k3s" 73 | port = 6443 74 | } 75 | 76 | depends_on = [google_compute_router_nat.nat] 77 | } 78 | -------------------------------------------------------------------------------- /k3s-servers/lb.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_health_check" "k3s-health-check-internal" { 2 | name = "k3s-servers-internal-hc" 3 | 4 | timeout_sec = 1 5 | check_interval_sec = 5 6 | 7 | tcp_health_check { 8 | port = 6443 9 | } 10 | } 11 | 12 | resource "google_compute_region_health_check" "k3s-health-check-external" { 13 | name = "k3s-servers-external-hc" 14 | region = var.region 15 | 16 | timeout_sec = 1 17 | check_interval_sec = 5 18 | 19 | tcp_health_check { 20 | port = 6443 21 | } 22 | } 23 | 24 | resource "google_compute_region_backend_service" "k3s-api-server-internal" { 25 | name = "k3s-api-server-internal" 26 | region = var.region 27 | load_balancing_scheme = "INTERNAL" 28 | health_checks = [google_compute_health_check.k3s-health-check-internal.id] 29 | backend { 30 | group = google_compute_region_instance_group_manager.k3s-servers.instance_group 31 | } 32 | } 33 | 34 | resource "google_compute_forwarding_rule" "k3s-api-server-internal" { 35 | name = "k3s-api-server-internal" 36 | region = var.region 37 | load_balancing_scheme = "INTERNAL" 38 | allow_global_access = true 39 | ip_address = google_compute_address.k3s-api-server-internal.address 40 | backend_service = google_compute_region_backend_service.k3s-api-server-internal.id 41 | ports = [6443] 42 | subnetwork = google_compute_subnetwork.k3s-servers.self_link 43 | } 44 | 45 | resource "google_compute_region_backend_service" "k3s-api-server-external" { 46 | name = "k3s-api-server-external" 47 | region = var.region 48 | load_balancing_scheme = "EXTERNAL" 49 | health_checks = [google_compute_region_health_check.k3s-health-check-external.id] 50 | backend { 51 | group = google_compute_region_instance_group_manager.k3s-servers.instance_group 52 | } 53 | } 54 | 55 | resource "google_compute_forwarding_rule" "k3s-api-server-external" { 56 | name = "k3s-api-server-external" 57 | region = var.region 58 | load_balancing_scheme = "EXTERNAL" 59 | ip_address = google_compute_address.k3s-api-server-external.address 60 | backend_service = google_compute_region_backend_service.k3s-api-server-external.id 61 | port_range = "6443-6443" 62 | } 63 | -------------------------------------------------------------------------------- /k3s-servers/network.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_subnetwork" "k3s-servers" { 2 | name = "k3s-servers" 3 | network = var.network 4 | region = var.region 5 | ip_cidr_range = var.cidr_range 6 | 7 | private_ip_google_access = true 8 | } 9 | 10 | resource "google_compute_address" "k3s-api-server-internal" { 11 | name = "k3s-api-server-internal" 12 | address_type = "INTERNAL" 13 | purpose = "GCE_ENDPOINT" 14 | region = var.region 15 | subnetwork = google_compute_subnetwork.k3s-servers.id 16 | } 17 | 18 | resource "google_compute_address" "k3s-api-server-external" { 19 | name = "k3s-api-server-external" 20 | region = var.region 21 | } 22 | 23 | resource "google_compute_firewall" "k3s-api-allow-hc" { 24 | name = "k3s-api-allow-hc" 25 | network = var.network 26 | source_ranges = ["130.211.0.0/22", "35.191.0.0/16", "209.85.152.0/22", "209.85.204.0/22"] 27 | allow { 28 | protocol = "tcp" 29 | ports = [6443] 30 | } 31 | target_tags = ["k3s-server"] 32 | direction = "INGRESS" 33 | } 34 | 35 | resource "google_compute_firewall" "k3s-api-authorized-networks" { 36 | name = "k3s-api-authorized-networks" 37 | network = var.network 38 | source_ranges = split(",", var.authorized_networks) 39 | allow { 40 | protocol = "tcp" 41 | ports = [6443] 42 | } 43 | target_tags = ["k3s-server"] 44 | direction = "INGRESS" 45 | } 46 | 47 | resource "google_compute_router" "router" { 48 | name = "k3s-servers" 49 | region = var.region 50 | network = var.network 51 | } 52 | 53 | resource "google_compute_router_nat" "nat" { 54 | name = "k3s-servers" 55 | router = google_compute_router.router.name 56 | region = var.region 57 | nat_ip_allocate_option = "AUTO_ONLY" 58 | source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" 59 | subnetwork { 60 | name = google_compute_subnetwork.k3s-servers.id 61 | source_ip_ranges_to_nat = ["ALL_IP_RANGES"] 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /k3s-servers/outputs.tf: -------------------------------------------------------------------------------- 1 | output "token" { 2 | value = random_string.token.result 3 | } 4 | 5 | output "internal_lb_ip_address" { 6 | value = google_compute_address.k3s-api-server-internal.address 7 | } 8 | 9 | output "external_lb_ip_address" { 10 | value = google_compute_address.k3s-api-server-external.address 11 | } 12 | -------------------------------------------------------------------------------- /k3s-servers/templates/server.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="v1.19.3+k3s3" sh -s - \ 4 | --write-kubeconfig-mode 644 \ 5 | --token "${token}" \ 6 | --tls-san "${internal_lb_ip_address}" \ 7 | --tls-san "${external_lb_ip_address}" \ 8 | --node-taint "CriticalAddonsOnly=true:NoExecute" \ 9 | --disable traefik \ 10 | --datastore-endpoint "postgres://${db_user}:${db_password}@${db_host}:5432/${db_name}" -------------------------------------------------------------------------------- /k3s-servers/variables.tf: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | } 4 | 5 | variable "network" { 6 | type = string 7 | } 8 | 9 | variable "region" { 10 | type = string 11 | } 12 | 13 | variable "cidr_range" { 14 | } 15 | 16 | variable "machine_type" { 17 | type = string 18 | } 19 | 20 | variable "target_size" { 21 | type = number 22 | default = 3 23 | } 24 | 25 | variable "authorized_networks" { 26 | type = string 27 | } 28 | 29 | variable "service_account" { 30 | type = string 31 | } 32 | 33 | variable "db_host" { 34 | type = string 35 | } 36 | 37 | variable "db_name" { 38 | type = string 39 | } 40 | 41 | variable "db_user" { 42 | type = string 43 | } 44 | 45 | variable "db_password" { 46 | type = string 47 | } 48 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13" 3 | 4 | required_providers { 5 | google = ">= 3.3" 6 | } 7 | } 8 | 9 | provider "google" { 10 | project = var.project 11 | } 12 | 13 | resource "google_compute_network" "k3s" { 14 | name = "k3s" 15 | auto_create_subnetworks = "false" 16 | } 17 | 18 | resource "google_compute_firewall" "k3s-firewall" { 19 | name = "k3s-allow-internal" 20 | network = google_compute_network.k3s.id 21 | allow { 22 | protocol = "all" 23 | } 24 | source_tags = ["k3s"] 25 | target_tags = ["k3s"] 26 | } 27 | 28 | resource "google_compute_firewall" "iap-firewall" { 29 | name = "iap-allow-ssh" 30 | network = google_compute_network.k3s.id 31 | 32 | allow { 33 | protocol = "tcp" 34 | ports = ["22"] 35 | } 36 | 37 | source_ranges = ["35.235.240.0/20"] 38 | target_tags = ["k3s"] 39 | } 40 | 41 | 42 | resource "google_service_account" "k3s-server" { 43 | account_id = "k3s-server" 44 | } 45 | 46 | resource "google_service_account" "k3s-agent" { 47 | account_id = "k3s-agent" 48 | } 49 | 50 | module "k3s-db" { 51 | source = "./k3s-db" 52 | 53 | project = var.project 54 | network = google_compute_network.k3s.self_link 55 | region = var.database.region 56 | db_tier = var.database.tier 57 | } 58 | 59 | module "k3s-servers" { 60 | source = "./k3s-servers" 61 | 62 | project = var.project 63 | network = google_compute_network.k3s.self_link 64 | region = var.servers.region 65 | cidr_range = var.servers.cidr_range 66 | machine_type = var.servers.machine_type 67 | target_size = var.servers.target_size 68 | authorized_networks = var.servers.authorized_networks 69 | service_account = google_service_account.k3s-server.email 70 | 71 | db_host = module.k3s-db.db_host 72 | db_name = module.k3s-db.db_name 73 | db_user = module.k3s-db.db_user 74 | db_password = module.k3s-db.db_password 75 | } 76 | 77 | module "k3s-agents" { 78 | source = "./k3s-agents" 79 | for_each = var.agents 80 | 81 | project = var.project 82 | name = each.key 83 | network = google_compute_network.k3s.self_link 84 | region = each.value.region 85 | cidr_range = each.value.cidr_range 86 | machine_type = each.value.machine_type 87 | target_size = each.value.target_size 88 | token = module.k3s-servers.token 89 | server_address = module.k3s-servers.internal_lb_ip_address 90 | service_account = google_service_account.k3s-agent.email 91 | } 92 | 93 | -------------------------------------------------------------------------------- /terraform.example.tfvars: -------------------------------------------------------------------------------- 1 | region = "europe-west1" 2 | 3 | database = { 4 | tier = "db-f1-micro" 5 | region = "europe-west1" 6 | } 7 | 8 | servers = { 9 | region = "europe-west1" 10 | cidr_range = "10.128.0.0/20" 11 | machine_type = "e2-micro" 12 | target_size = 3 13 | authorized_networks = "91.183.51.235/32" 14 | } 15 | 16 | agents = { 17 | eu001 = { 18 | region = "europe-west4", 19 | cidr_range = "10.164.0.0/20" 20 | machine_type = "e2-micro" 21 | target_size = 2 22 | }, 23 | eu002 = { 24 | region = "europe-north1", 25 | cidr_range = "10.166.0.0/20" 26 | machine_type = "e2-micro" 27 | target_size = 2 28 | }, 29 | eu003 = { 30 | region = "europe-west2", 31 | cidr_range = "10.154.0.0/20" 32 | machine_type = "e2-medium" 33 | target_size = 2 34 | }, 35 | } -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | } 4 | 5 | variable "region" { 6 | type = string 7 | } 8 | 9 | variable database { 10 | type = map 11 | } 12 | 13 | variable servers { 14 | type = map 15 | } 16 | 17 | variable agents { 18 | type = map 19 | } 20 | --------------------------------------------------------------------------------