├── packer ├── justfile └── hcloud.pkr.hcl ├── modules ├── hcloud-apply │ ├── outputs.tf │ ├── terraform.tf │ ├── ssh.tf │ ├── variables.tf │ ├── servers.tf │ └── firewalls.tf ├── gcp-wif-apply │ ├── outputs.tf │ ├── terraform.tf │ ├── certificates.tf │ ├── file-jwks.tf │ ├── file-openid-configuration.tf │ └── variables.tf ├── gcp-wif │ ├── key.tf │ ├── pool.tf │ ├── terraform.tf │ ├── outputs.tf │ ├── pool-provider-oidc.tf │ ├── bucket.tf │ ├── variables.tf │ ├── service-accounts.tf │ └── locals.tf ├── talos-cluster │ ├── machine-secrets.tf │ ├── terraform.tf │ ├── client-configuration.tf │ ├── machine-configurations.tf │ ├── outputs.tf │ ├── patches │ │ ├── control-planes.yaml │ │ └── common.yaml │ ├── variables.tf │ └── locals.tf ├── talos-apply │ ├── terraform.tf │ ├── bootstrap.tf │ ├── config.tf │ ├── apply.tf │ ├── health.tf │ ├── variables.tf │ └── outputs.tf └── hcloud-pool │ ├── terraform.tf │ ├── outputs.tf │ ├── variables.tf │ ├── main.tf │ └── locals.tf ├── manifests ├── argocd │ ├── namespace.yaml │ ├── patch-argocd-cm.yaml │ └── kustomization.yaml ├── gcp-wif-webhook │ ├── namespace.yaml │ ├── kustomization.yaml │ └── patch-deployment.yaml ├── iperf │ ├── kustomization.yaml │ ├── client.yaml │ └── server.yaml ├── cilium │ ├── patch-namespaces.yaml │ └── kustomization.yaml ├── talos-ccm │ ├── patch-config-map.yaml │ └── kustomization.yaml ├── hcloud-ccm │ └── kustomization.yaml └── hcloud-csi │ └── kustomization.yaml ├── dev ├── locals.tf ├── providers.tf ├── 0-gcp.tf ├── justfile ├── terraform.tf ├── 0-hcloud.tf ├── manifests │ └── external-secrets.yaml ├── 1-gcp-wif.tf ├── 1-talos.tf └── .terraform.lock.hcl ├── .gitignore ├── CHANGELOG.md ├── examples ├── justfile ├── minimal.tf └── multi-region.tf ├── justfile ├── LICENSE └── README.md /packer/justfile: -------------------------------------------------------------------------------- 1 | init: 2 | packer init . 3 | build: 4 | packer build . -------------------------------------------------------------------------------- /modules/hcloud-apply/outputs.tf: -------------------------------------------------------------------------------- 1 | output "MODULE_NAME" { 2 | value = "hcloud-apply" 3 | } 4 | -------------------------------------------------------------------------------- /modules/gcp-wif-apply/outputs.tf: -------------------------------------------------------------------------------- 1 | output "MODULE_NAME" { 2 | value = "gcp-wif-apply" 3 | } 4 | -------------------------------------------------------------------------------- /manifests/argocd/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: argocd 5 | -------------------------------------------------------------------------------- /modules/gcp-wif/key.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "this" { 2 | algorithm = "RSA" 3 | rsa_bits = 4096 4 | } 5 | -------------------------------------------------------------------------------- /dev/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | hcloud_token = data.google_secret_manager_secret_version.hcloud_token.secret_data 3 | } 4 | -------------------------------------------------------------------------------- /manifests/gcp-wif-webhook/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: gcp-wif-webhook-system 5 | -------------------------------------------------------------------------------- /modules/gcp-wif/pool.tf: -------------------------------------------------------------------------------- 1 | resource "google_iam_workload_identity_pool" "this" { 2 | workload_identity_pool_id = var.name 3 | } 4 | -------------------------------------------------------------------------------- /modules/talos-cluster/machine-secrets.tf: -------------------------------------------------------------------------------- 1 | resource "talos_machine_secrets" "this" { 2 | talos_version = var.talos_version 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # manifests 2 | charts 3 | .build 4 | 5 | # terraform 6 | .terraform 7 | current-plan 8 | talos-config 9 | kube-config 10 | support* 11 | -------------------------------------------------------------------------------- /modules/talos-apply/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | talos = { 4 | source = "siderolabs/talos" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /modules/talos-cluster/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | talos = { 4 | source = "siderolabs/talos" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /manifests/iperf/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - server.yaml 5 | - client.yaml 6 | -------------------------------------------------------------------------------- /modules/hcloud-apply/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | hcloud = { 4 | source = "hetznercloud/hcloud" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /modules/hcloud-pool/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | hcloud = { 4 | source = "hetznercloud/hcloud" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dev/providers.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = "miran248-talos-modules-dev" 3 | region = "global" 4 | } 5 | provider "hcloud" { 6 | token = local.hcloud_token 7 | } 8 | -------------------------------------------------------------------------------- /manifests/cilium/patch-namespaces.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-spire 5 | labels: 6 | pod-security.kubernetes.io/enforce: privileged 7 | -------------------------------------------------------------------------------- /dev/0-gcp.tf: -------------------------------------------------------------------------------- 1 | # dns 2 | data "google_dns_managed_zone" "this" { 3 | name = "dev" 4 | } 5 | 6 | # secrets 7 | data "google_secret_manager_secret_version" "hcloud_token" { 8 | secret = "hcloud" 9 | } 10 | -------------------------------------------------------------------------------- /modules/gcp-wif/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | google = { 4 | source = "hashicorp/google" 5 | } 6 | tls = { 7 | source = "hashicorp/tls" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /manifests/talos-ccm/patch-config-map.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: talos-ccm 5 | namespace: kube-system 6 | data: 7 | ccm-config.yaml: | 8 | global: 9 | preferIPv6: true 10 | -------------------------------------------------------------------------------- /modules/hcloud-apply/ssh.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "ssh_key" { 2 | algorithm = "ED25519" 3 | } 4 | resource "hcloud_ssh_key" "this" { 5 | name = var.pool.prefix 6 | public_key = tls_private_key.ssh_key.public_key_openssh 7 | } 8 | -------------------------------------------------------------------------------- /modules/talos-cluster/client-configuration.tf: -------------------------------------------------------------------------------- 1 | data "talos_client_configuration" "this" { 2 | client_configuration = talos_machine_secrets.this.client_configuration 3 | cluster_name = var.name 4 | endpoints = [var.endpoint] 5 | } 6 | -------------------------------------------------------------------------------- /modules/talos-apply/bootstrap.tf: -------------------------------------------------------------------------------- 1 | resource "talos_machine_bootstrap" "this" { 2 | client_configuration = var.cluster.machine_secrets.client_configuration 3 | endpoint = var.cluster.endpoint 4 | node = var.cluster.names.control_planes[0] 5 | } 6 | -------------------------------------------------------------------------------- /modules/gcp-wif-apply/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | google = { 4 | source = "hashicorp/google" 5 | } 6 | local = { 7 | source = "hashicorp/local" 8 | } 9 | terracurl = { 10 | source = "devops-rob/terracurl" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 4 | 5 | ### [3.2.3](https://github.com/miran248/terraform-talos-modules/compare/v3.2.2...v3.2.3) (2025-12-04) 6 | -------------------------------------------------------------------------------- /examples/justfile: -------------------------------------------------------------------------------- 1 | apply: 2 | terraform init -upgrade 3 | terraform apply -auto-approve 4 | rm -f talos-config kube-config 5 | terraform output --raw talos_config > talos-config 6 | TALOSCONFIG=talos-config talosctl -n c1 kubeconfig kube-config 7 | 8 | destroy: 9 | rm -f talos-config kube-config 10 | terraform destroy -auto-approve 11 | -------------------------------------------------------------------------------- /manifests/argocd/patch-argocd-cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: argocd-cm 5 | namespace: argocd 6 | data: 7 | kustomize.buildOptions: --enable-helm 8 | resource.exclusions: | 9 | - apiGroups: 10 | - cilium.io 11 | kinds: 12 | - CiliumIdentity 13 | clusters: 14 | - "*" 15 | -------------------------------------------------------------------------------- /modules/talos-apply/config.tf: -------------------------------------------------------------------------------- 1 | resource "talos_cluster_kubeconfig" "this" { 2 | client_configuration = var.cluster.machine_secrets.client_configuration 3 | endpoint = var.cluster.endpoint 4 | node = var.cluster.names.control_planes[0] 5 | 6 | depends_on = [ 7 | talos_machine_bootstrap.this, 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /dev/justfile: -------------------------------------------------------------------------------- 1 | apply: 2 | terraform init -upgrade 3 | terraform apply -auto-approve 4 | rm -f ../talos-config ../kube-config 5 | terraform output --raw talos_config > ../talos-config 6 | TALOSCONFIG=../talos-config talosctl -n c1 kubeconfig ../kube-config 7 | 8 | destroy: 9 | rm -f ../talos-config ../kube-config 10 | terraform destroy -auto-approve 11 | -------------------------------------------------------------------------------- /dev/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | cloud { 3 | organization = "miran248" 4 | 5 | workspaces { 6 | name = "dev" 7 | } 8 | } 9 | required_providers { 10 | google = { 11 | source = "hashicorp/google" 12 | } 13 | hcloud = { 14 | source = "hetznercloud/hcloud" 15 | } 16 | } 17 | required_version = ">= 1" 18 | } 19 | -------------------------------------------------------------------------------- /dev/0-hcloud.tf: -------------------------------------------------------------------------------- 1 | data "hcloud_image" "v1_9_5_amd64" { 2 | with_selector = "name=talos,version=v1.9.5,arch=amd64" 3 | } 4 | data "hcloud_image" "v1_10_6_amd64" { 5 | with_selector = "name=talos,version=v1.10.6,arch=amd64" 6 | } 7 | data "hcloud_datacenter" "nuremberg" { 8 | name = "nbg1-dc3" 9 | } 10 | data "hcloud_datacenter" "helsinki" { 11 | name = "hel1-dc2" 12 | } 13 | -------------------------------------------------------------------------------- /modules/gcp-wif/outputs.tf: -------------------------------------------------------------------------------- 1 | output "MODULE_NAME" { 2 | value = "gcp-wif" 3 | } 4 | 5 | output "name" { 6 | value = var.name 7 | } 8 | output "bucket_name" { 9 | value = var.bucket_name 10 | } 11 | output "bucket_location" { 12 | value = var.bucket_location 13 | } 14 | 15 | output "ids" { 16 | value = local.ids 17 | } 18 | 19 | output "patches" { 20 | value = local.patches 21 | } 22 | -------------------------------------------------------------------------------- /manifests/hcloud-ccm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | helmCharts: 4 | - name: hcloud-cloud-controller-manager 5 | repo: https://charts.hetzner.cloud 6 | version: 1.26.0 7 | releaseName: hcloud-ccm 8 | namespace: kube-system 9 | includeCRDs: true 10 | valuesInline: 11 | nameOverride: hcloud-ccm 12 | networking: 13 | enabled: false 14 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | @build: 2 | mkdir -p .build/manifests 3 | just build-helm manifests/argocd 4 | just build-helm manifests/cilium 5 | just build-helm manifests/gcp-wif-webhook 6 | just build-helm manifests/hcloud-ccm 7 | just build-helm manifests/hcloud-csi 8 | just build-helm manifests/iperf 9 | just build-helm manifests/talos-ccm 10 | 11 | build-helm NAME: 12 | kustomize build --enable-helm {{ NAME }} > .build/{{ NAME }}.yaml 13 | -------------------------------------------------------------------------------- /modules/talos-apply/apply.tf: -------------------------------------------------------------------------------- 1 | resource "talos_machine_configuration_apply" "this" { 2 | for_each = var.cluster.configs 3 | client_configuration = var.cluster.machine_secrets.client_configuration 4 | endpoint = var.cluster.endpoint 5 | machine_configuration_input = each.value 6 | node = each.key 7 | 8 | depends_on = [ 9 | talos_machine_bootstrap.this, 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /modules/talos-apply/health.tf: -------------------------------------------------------------------------------- 1 | # data "talos_cluster_health" "this" { 2 | # client_configuration = var.cluster.machine_secrets.client_configuration 3 | # control_plane_nodes = var.cluster.names.control_planes 4 | # worker_nodes = var.cluster.names.workers 5 | # endpoints = [var.cluster.endpoint] 6 | # # skip_kubernetes_checks = true 7 | 8 | # depends_on = [ 9 | # talos_machine_bootstrap.this, 10 | # ] 11 | # } 12 | -------------------------------------------------------------------------------- /modules/gcp-wif-apply/certificates.tf: -------------------------------------------------------------------------------- 1 | resource "local_sensitive_file" "ca_certificate" { 2 | filename = "${path.module}/ca_certificate" 3 | content = var.apply.ca_certificate 4 | } 5 | resource "local_sensitive_file" "client_certificate" { 6 | filename = "${path.module}/client_certificate" 7 | content = var.apply.client_certificate 8 | } 9 | resource "local_sensitive_file" "client_key" { 10 | filename = "${path.module}/client_key" 11 | content = var.apply.client_key 12 | } 13 | -------------------------------------------------------------------------------- /modules/talos-cluster/machine-configurations.tf: -------------------------------------------------------------------------------- 1 | data "talos_machine_configuration" "this" { 2 | for_each = local.nodes 3 | cluster_endpoint = local.cluster_endpoint 4 | cluster_name = var.name 5 | config_patches = each.value.patches 6 | kubernetes_version = var.kubernetes_version 7 | machine_secrets = talos_machine_secrets.this.machine_secrets 8 | machine_type = each.value.talos.machine_type 9 | talos_version = var.talos_version 10 | } 11 | -------------------------------------------------------------------------------- /manifests/argocd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - namespace.yaml 5 | patches: 6 | - path: patch-argocd-cm.yaml 7 | helmCharts: 8 | - name: argo-cd 9 | repo: https://argoproj.github.io/argo-helm 10 | version: 8.3.0 11 | releaseName: argocd 12 | namespace: argocd 13 | includeCRDs: true 14 | valuesInline: 15 | dex: 16 | enabled: false 17 | configs: 18 | params: 19 | server.insecure: "true" 20 | -------------------------------------------------------------------------------- /modules/gcp-wif/pool-provider-oidc.tf: -------------------------------------------------------------------------------- 1 | resource "google_iam_workload_identity_pool_provider" "oidc" { 2 | workload_identity_pool_id = google_iam_workload_identity_pool.this.workload_identity_pool_id 3 | workload_identity_pool_provider_id = "${var.name}-oidc" 4 | 5 | attribute_mapping = { 6 | "google.subject" = "assertion.sub" 7 | "attribute.sub" = "assertion.sub" 8 | } 9 | 10 | oidc { 11 | issuer_uri = local.oidc_bucket_url 12 | allowed_audiences = [ 13 | "sts.googleapis.com", 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dev/manifests/external-secrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ClusterSecretStore 3 | metadata: 4 | name: google 5 | spec: 6 | provider: 7 | gcpsm: 8 | projectID: miran248-talos-modules-dev 9 | --- 10 | apiVersion: external-secrets.io/v1beta1 11 | kind: ExternalSecret 12 | metadata: 13 | name: hcloud 14 | namespace: kube-system 15 | spec: 16 | refreshInterval: 1h 17 | secretStoreRef: 18 | kind: ClusterSecretStore 19 | name: google 20 | data: 21 | - secretKey: token 22 | remoteRef: 23 | key: hcloud 24 | -------------------------------------------------------------------------------- /modules/gcp-wif/bucket.tf: -------------------------------------------------------------------------------- 1 | resource "google_storage_bucket" "oidc" { 2 | name = var.bucket_name 3 | location = var.bucket_location 4 | force_destroy = true 5 | storage_class = "NEARLINE" 6 | 7 | uniform_bucket_level_access = true 8 | 9 | soft_delete_policy { 10 | retention_duration_seconds = 0 11 | } 12 | 13 | versioning { 14 | enabled = false 15 | } 16 | } 17 | resource "google_storage_bucket_iam_member" "oidc" { 18 | bucket = google_storage_bucket.oidc.name 19 | role = "roles/storage.objectViewer" 20 | member = "allUsers" 21 | } 22 | -------------------------------------------------------------------------------- /modules/hcloud-pool/outputs.tf: -------------------------------------------------------------------------------- 1 | output "MODULE_NAME" { 2 | value = "hcloud-pool" 3 | } 4 | 5 | output "prefix" { 6 | value = var.prefix 7 | } 8 | output "datacenter" { 9 | value = var.datacenter 10 | } 11 | 12 | output "cidr" { 13 | value = var.cidr 14 | } 15 | output "load_balancer_ip" { 16 | value = var.load_balancer_ip 17 | } 18 | 19 | output "ids" { 20 | value = local.ids 21 | } 22 | 23 | output "control_planes" { 24 | value = local.control_planes 25 | } 26 | output "workers" { 27 | value = local.workers 28 | } 29 | output "nodes" { 30 | value = local.nodes 31 | } 32 | -------------------------------------------------------------------------------- /manifests/iperf/client.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: iperf-client 5 | spec: 6 | securityContext: 7 | runAsNonRoot: true 8 | runAsUser: 1000 9 | seccompProfile: 10 | type: RuntimeDefault 11 | containers: 12 | - name: iperf 13 | image: cagedata/iperf3 14 | command: ["sleep", "30000"] 15 | securityContext: 16 | allowPrivilegeEscalation: false 17 | capabilities: 18 | drop: 19 | - ALL 20 | readOnlyRootFilesystem: false 21 | runAsNonRoot: true 22 | seccompProfile: 23 | type: RuntimeDefault 24 | -------------------------------------------------------------------------------- /manifests/gcp-wif-webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - namespace.yaml 5 | patches: 6 | - path: patch-deployment.yaml 7 | helmCharts: 8 | - name: gcp-workload-identity-federation-webhook 9 | repo: https://pfnet-research.github.io/gcp-workload-identity-federation-webhook 10 | version: 0.5.0 11 | releaseName: gcp-wif-webhook 12 | namespace: gcp-wif-webhook-system 13 | includeCRDs: true 14 | valuesInline: 15 | nameOverride: gcp-wif-webhook 16 | controllerManager: 17 | manager: 18 | args: 19 | - --token-default-mode=0444 20 | -------------------------------------------------------------------------------- /modules/talos-apply/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster" { 2 | type = object({ 3 | MODULE_NAME = string 4 | endpoint = string 5 | machine_secrets = object({ 6 | client_configuration = object({ ca_certificate = string, client_certificate = string, client_key = string }) 7 | }) 8 | names = object({ 9 | control_planes = list(string) 10 | workers = list(string) 11 | }) 12 | configs = map(string) 13 | }) 14 | description = "talos-cluster module outputs" 15 | validation { 16 | condition = var.cluster.MODULE_NAME == "talos-cluster" 17 | error_message = "must be of type talos-cluster" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /manifests/iperf/server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: iperf-server 5 | spec: 6 | securityContext: 7 | runAsNonRoot: true 8 | runAsUser: 1000 9 | seccompProfile: 10 | type: RuntimeDefault 11 | containers: 12 | - name: iperf 13 | image: cagedata/iperf3 14 | command: ["iperf3", "-s"] 15 | ports: 16 | - containerPort: 5201 17 | securityContext: 18 | allowPrivilegeEscalation: false 19 | capabilities: 20 | drop: 21 | - ALL 22 | readOnlyRootFilesystem: false 23 | runAsNonRoot: true 24 | seccompProfile: 25 | type: RuntimeDefault 26 | -------------------------------------------------------------------------------- /modules/gcp-wif/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | type = string 3 | description = "name of the google workload identity pool" 4 | } 5 | variable "bucket_name" { 6 | type = string 7 | description = "name of the google object store bucket" 8 | } 9 | variable "bucket_location" { 10 | type = string 11 | description = "location of the google object store bucket" 12 | } 13 | 14 | variable "service_accounts" { 15 | type = list(object({ 16 | subject = string 17 | name = string 18 | roles = list(string) 19 | })) 20 | description = "list of existing kubernetes service accounts (subjects, format `namespace:name`) to be created on google" 21 | } 22 | -------------------------------------------------------------------------------- /dev/1-gcp-wif.tf: -------------------------------------------------------------------------------- 1 | module "gcp_wif" { 2 | source = "../modules/gcp-wif" 3 | 4 | name = "talos" 5 | bucket_name = "miran248-talos-modules-dev-wif" 6 | bucket_location = "EUROPE-WEST3" 7 | 8 | service_accounts = [ 9 | { subject = "cert-manager:cert-manager", name = "cert-manager", roles = ["roles/dns.admin"] }, 10 | { subject = "external-dns:external-dns", name = "external-dns", roles = ["roles/dns.admin"] }, 11 | 12 | { subject = "external-secrets:external-secrets", name = "external-secrets", roles = [ 13 | "roles/iam.serviceAccountTokenCreator", 14 | "roles/secretmanager.admin", 15 | # "roles/secretmanager.secretAccessor", 16 | ] }, 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /modules/talos-apply/outputs.tf: -------------------------------------------------------------------------------- 1 | output "MODULE_NAME" { 2 | value = "talos-apply" 3 | } 4 | 5 | output "kube_config" { 6 | value = talos_cluster_kubeconfig.this.kubeconfig_raw 7 | sensitive = true 8 | } 9 | 10 | output "ca_certificate" { 11 | value = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate) 12 | sensitive = true 13 | } 14 | output "client_certificate" { 15 | value = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_certificate) 16 | sensitive = true 17 | } 18 | output "client_key" { 19 | value = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_key) 20 | sensitive = true 21 | } 22 | -------------------------------------------------------------------------------- /modules/talos-cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | output "MODULE_NAME" { 2 | value = "talos-cluster" 3 | } 4 | 5 | output "name" { 6 | value = var.name 7 | } 8 | output "endpoint" { 9 | value = var.endpoint 10 | } 11 | 12 | output "cluster_endpoint" { 13 | value = local.cluster_endpoint 14 | } 15 | output "machine_secrets" { 16 | value = talos_machine_secrets.this 17 | } 18 | output "talos_config" { 19 | value = data.talos_client_configuration.this.talos_config 20 | sensitive = true 21 | } 22 | 23 | output "nodes" { 24 | value = local.nodes 25 | } 26 | 27 | output "names" { 28 | value = local.names 29 | } 30 | output "public_ips6" { 31 | value = local.public_ips6 32 | } 33 | output "configs" { 34 | value = local.configs 35 | } 36 | -------------------------------------------------------------------------------- /manifests/gcp-wif-webhook/patch-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: gcp-wif-webhook-controller-manager 5 | namespace: gcp-wif-webhook-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | securityContext: 12 | seccompProfile: 13 | type: RuntimeDefault 14 | allowPrivilegeEscalation: false 15 | capabilities: 16 | drop: 17 | - ALL 18 | - name: kube-rbac-proxy 19 | securityContext: 20 | seccompProfile: 21 | type: RuntimeDefault 22 | allowPrivilegeEscalation: false 23 | capabilities: 24 | drop: 25 | - ALL 26 | securityContext: 27 | runAsNonRoot: true 28 | -------------------------------------------------------------------------------- /modules/gcp-wif/service-accounts.tf: -------------------------------------------------------------------------------- 1 | data "google_project" "this" { 2 | } 3 | 4 | resource "google_service_account" "this" { 5 | for_each = local.service_accounts 6 | account_id = each.value.name 7 | } 8 | resource "google_project_iam_member" "this" { 9 | for_each = local.roles 10 | project = data.google_project.this.project_id 11 | member = google_service_account.this[each.value.name].member 12 | role = each.value.role 13 | } 14 | resource "google_service_account_iam_member" "this" { 15 | for_each = local.service_accounts 16 | service_account_id = google_service_account.this[each.key].name 17 | role = "roles/iam.workloadIdentityUser" 18 | member = "principal://iam.googleapis.com/${google_iam_workload_identity_pool.this.name}/subject/system:serviceaccount:${each.value.subject}" 19 | } 20 | -------------------------------------------------------------------------------- /modules/gcp-wif-apply/file-jwks.tf: -------------------------------------------------------------------------------- 1 | data "terracurl_request" "jwks" { 2 | name = "jwks" 3 | url = "${var.cluster.cluster_endpoint}/openid/v1/jwks" 4 | method = "GET" 5 | 6 | ca_cert_file = local_sensitive_file.ca_certificate.filename 7 | cert_file = local_sensitive_file.client_certificate.filename 8 | key_file = local_sensitive_file.client_key.filename 9 | skip_tls_verify = false 10 | 11 | response_codes = [200] 12 | } 13 | 14 | resource "terraform_data" "jwks" { 15 | input = timestamp() 16 | } 17 | resource "google_storage_bucket_object" "jwks" { 18 | bucket = var.identities.ids.oidc_bucket 19 | name = "openid/v1/jwks" 20 | content = data.terracurl_request.jwks.response 21 | cache_control = "public, max-age=0" 22 | content_type = "application/json" 23 | 24 | lifecycle { 25 | replace_triggered_by = [ 26 | terraform_data.jwks, 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /manifests/talos-ccm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | helmCharts: 4 | - name: talos-cloud-controller-manager 5 | repo: oci://ghcr.io/siderolabs/charts 6 | version: 0.5.0 7 | releaseName: talos-ccm 8 | namespace: kube-system 9 | includeCRDs: true 10 | valuesInline: 11 | # logVerbosityLevel: 5 12 | 13 | nameOverride: talos-ccm 14 | daemonSet: 15 | enabled: true 16 | # features: 17 | # # required when using ipv6 stack, otherwise ccm picks hetzner's cgnat ip address.. 18 | # preferIPv6: true 19 | enabledControllers: 20 | - cloud-node 21 | - node-csr-approval 22 | - node-ipam-controller 23 | extraArgs: 24 | - --allocate-node-cidrs 25 | - --cidr-allocator-type=CloudAllocator 26 | - --node-cidr-mask-size-ipv6=112 27 | patches: 28 | - path: patch-config-map.yaml 29 | -------------------------------------------------------------------------------- /modules/hcloud-apply/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster" { 2 | type = object({ 3 | nodes = map(object({ 4 | public_ip6_network_64 = string 5 | })) 6 | configs = map(string) 7 | }) 8 | description = "talos-cluster module outputs" 9 | } 10 | variable "pool" { 11 | type = object({ 12 | MODULE_NAME = string 13 | prefix = string 14 | datacenter = string 15 | ids = object({ 16 | network = optional(string) 17 | load_balancer = optional(string) 18 | }) 19 | nodes = map(object({ 20 | name = string 21 | server_type = string 22 | image_id = number 23 | private_ip4 = optional(string) 24 | public_ip6_id = number 25 | public_ip6_network_64 = string 26 | public_ip6 = string 27 | })) 28 | }) 29 | description = "hcloud-pool module outputs" 30 | validation { 31 | condition = var.pool.MODULE_NAME == "hcloud-pool" 32 | error_message = "must be of type hcloud-pool" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /packer/hcloud.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | hcloud = { 4 | source = "github.com/hetznercloud/hcloud" 5 | version = ">= 1.0.0" 6 | } 7 | } 8 | } 9 | 10 | source "hcloud" "talos-amd64" { 11 | image = "debian-12" 12 | location = "fsn1" 13 | rescue = "linux64" 14 | server_type = "cx22" 15 | ssh_username = "root" 16 | snapshot_name = "talos-v1.10.6-amd64" 17 | snapshot_labels = { 18 | name = "talos" 19 | version = "v1.10.6" 20 | arch = "amd64" 21 | } 22 | } 23 | 24 | build { 25 | sources = ["source.hcloud.talos-amd64"] 26 | provisioner "shell" { 27 | inline = [ 28 | "apt-get install -y wget", 29 | # https://factory.talos.dev/?arch=amd64&cmdline-set=true&extensions=-&platform=hcloud&target=cloud&version=1.10.6 30 | "wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/v1.10.6/hcloud-amd64.raw.xz", 31 | "xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync", 32 | ] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /modules/gcp-wif-apply/file-openid-configuration.tf: -------------------------------------------------------------------------------- 1 | data "terracurl_request" "openid_configuration" { 2 | name = "openid-configuration" 3 | url = "${var.cluster.cluster_endpoint}/.well-known/openid-configuration" 4 | method = "GET" 5 | 6 | ca_cert_file = local_sensitive_file.ca_certificate.filename 7 | cert_file = local_sensitive_file.client_certificate.filename 8 | key_file = local_sensitive_file.client_key.filename 9 | skip_tls_verify = false 10 | 11 | response_codes = [200] 12 | } 13 | 14 | resource "terraform_data" "openid_configuration" { 15 | input = timestamp() 16 | } 17 | resource "google_storage_bucket_object" "openid_configuration" { 18 | bucket = var.identities.ids.oidc_bucket 19 | name = ".well-known/openid-configuration" 20 | content = data.terracurl_request.openid_configuration.response 21 | cache_control = "public, max-age=0" 22 | content_type = "application/json" 23 | 24 | lifecycle { 25 | replace_triggered_by = [ 26 | terraform_data.openid_configuration, 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /modules/gcp-wif-apply/variables.tf: -------------------------------------------------------------------------------- 1 | variable "identities" { 2 | type = object({ 3 | MODULE_NAME = string 4 | ids = object({ oidc_bucket = string }) 5 | }) 6 | description = "gcp-wif module outputs" 7 | validation { 8 | condition = var.identities.MODULE_NAME == "gcp-wif" 9 | error_message = "must be of type gcp-wif" 10 | } 11 | } 12 | variable "cluster" { 13 | type = object({ 14 | MODULE_NAME = string 15 | cluster_endpoint = string 16 | }) 17 | description = "talos-cluster module outputs" 18 | validation { 19 | condition = var.cluster.MODULE_NAME == "talos-cluster" 20 | error_message = "must be of type talos-cluster" 21 | } 22 | } 23 | variable "apply" { 24 | type = object({ 25 | MODULE_NAME = string 26 | ca_certificate = string 27 | client_certificate = string 28 | client_key = string 29 | }) 30 | description = "talos-apply module outputs" 31 | validation { 32 | condition = var.apply.MODULE_NAME == "talos-apply" 33 | error_message = "must be of type talos-apply" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /modules/gcp-wif/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | oidc_bucket_url = "https://storage.googleapis.com/${google_storage_bucket.oidc.id}" 3 | 4 | service_accounts = { for a in var.service_accounts : a.name => a } 5 | 6 | roles = merge(flatten([ 7 | for name, a in local.service_accounts : [ 8 | for role in a.roles : { 9 | "${join("-", [name, role])}" : { 10 | name = name 11 | role = role 12 | } 13 | } 14 | ] 15 | ])...) 16 | 17 | ids = { 18 | oidc_bucket = google_storage_bucket.oidc.id 19 | } 20 | 21 | patches = { 22 | control_planes = [ 23 | <<-EOF 24 | cluster: 25 | apiServer: 26 | extraArgs: 27 | api-audiences: "https://kubernetes.default.svc.cluster.local,iam.googleapis.com/${google_iam_workload_identity_pool_provider.oidc.name}" 28 | service-account-issuer: "${local.oidc_bucket_url}" 29 | service-account-jwks-uri: "${local.oidc_bucket_url}/openid/v1/jwks" 30 | serviceAccount: 31 | key: "${base64encode(tls_private_key.this.private_key_pem)}" 32 | EOF 33 | ] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Miran 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /modules/talos-cluster/patches/control-planes.yaml: -------------------------------------------------------------------------------- 1 | cluster: 2 | apiServer: 3 | # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ 4 | extraArgs: 5 | advertise-address: "::" 6 | bind-address: "::" 7 | controllerManager: 8 | # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ 9 | extraArgs: 10 | allocate-node-cidrs: false 11 | bind-address: "::" 12 | cloud-provider: external 13 | controllers: "*,tokencleaner,-node-ipam-controller" 14 | node-cidr-mask-size-ipv6: 112 15 | discovery: 16 | enabled: true 17 | registries: 18 | kubernetes: 19 | disabled: true 20 | service: 21 | disabled: false 22 | etcd: 23 | # https://etcd.io/docs/v3.5/op-guide/configuration/ 24 | extraArgs: 25 | listen-metrics-urls: "http://[::]:2381" 26 | externalCloudProvider: 27 | enabled: true 28 | scheduler: 29 | # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/ 30 | extraArgs: 31 | bind-address: "::" 32 | machine: 33 | features: 34 | kubernetesTalosAPIAccess: 35 | enabled: true 36 | allowedRoles: 37 | - "os:reader" 38 | allowedKubernetesNamespaces: 39 | - kube-system 40 | -------------------------------------------------------------------------------- /modules/hcloud-apply/servers.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_server" "this" { 2 | for_each = var.pool.nodes 3 | name = var.pool.nodes[each.key].name 4 | image = var.pool.nodes[each.key].image_id 5 | server_type = var.pool.nodes[each.key].server_type 6 | datacenter = var.pool.datacenter 7 | user_data = var.cluster.configs[each.key] 8 | ssh_keys = [ 9 | hcloud_ssh_key.this.id, 10 | ] 11 | 12 | delete_protection = false 13 | shutdown_before_deletion = true 14 | 15 | ignore_remote_firewall_ids = true 16 | firewall_ids = [ 17 | hcloud_firewall.deny_all.id, 18 | ] 19 | 20 | dynamic "network" { 21 | for_each = var.pool.nodes[each.key].private_ip4 == null ? [] : [1] 22 | content { 23 | network_id = var.pool.ids.network 24 | ip = var.pool.nodes[each.key].private_ip4 25 | alias_ips = [] 26 | } 27 | } 28 | 29 | public_net { 30 | ipv6_enabled = true 31 | ipv6 = var.pool.nodes[each.key].public_ip6_id 32 | ipv4_enabled = false 33 | } 34 | 35 | lifecycle { 36 | ignore_changes = [ 37 | image, 38 | user_data, 39 | ] 40 | } 41 | } 42 | 43 | resource "hcloud_firewall_attachment" "this" { 44 | firewall_id = hcloud_firewall.this.id 45 | server_ids = [for key, server in hcloud_server.this : server.id] 46 | } 47 | -------------------------------------------------------------------------------- /modules/hcloud-pool/variables.tf: -------------------------------------------------------------------------------- 1 | variable "prefix" { 2 | type = string 3 | description = "hcloud resource name prefix, must be unique" 4 | } 5 | variable "datacenter" { 6 | type = string 7 | description = "hcloud datacenter" 8 | } 9 | 10 | variable "cidr" { 11 | type = string 12 | description = "hcloud private network cidr4" 13 | nullable = true 14 | default = null 15 | } 16 | variable "load_balancer_ip" { 17 | type = string 18 | description = "hcloud load balancer ip4" 19 | nullable = true 20 | default = null 21 | } 22 | 23 | variable "control_planes" { 24 | type = list(object({ 25 | server_type = string 26 | image_id = number 27 | aliases = optional(list(string), []) 28 | patches = optional(list(string), []) 29 | removed = optional(bool, false) 30 | })) 31 | description = "define control planes" 32 | default = [] 33 | } 34 | variable "workers" { 35 | type = list(object({ 36 | server_type = string 37 | image_id = number 38 | aliases = optional(list(string), []) 39 | patches = optional(list(string), []) 40 | removed = optional(bool, false) 41 | })) 42 | description = "define workers" 43 | default = [] 44 | } 45 | 46 | variable "patches" { 47 | type = object({ 48 | common = optional(list(string), []) 49 | control_planes = optional(list(string), []) 50 | workers = optional(list(string), []) 51 | }) 52 | description = "pool-wide config patches" 53 | default = {} 54 | } 55 | -------------------------------------------------------------------------------- /modules/hcloud-pool/main.tf: -------------------------------------------------------------------------------- 1 | # data centers 2 | data "hcloud_datacenter" "this" { 3 | name = var.datacenter 4 | } 5 | data "hcloud_location" "this" { 6 | name = data.hcloud_datacenter.this.location.name 7 | } 8 | 9 | # ips 10 | resource "hcloud_primary_ip" "ips6" { 11 | for_each = local.s3.nodes 12 | name = "${each.value.name}-ip6" 13 | datacenter = data.hcloud_datacenter.this.name 14 | type = "ipv6" 15 | assignee_type = "server" 16 | auto_delete = false 17 | 18 | delete_protection = false 19 | } 20 | 21 | # networks 22 | resource "hcloud_network" "this" { 23 | count = var.cidr == null ? 0 : 1 24 | name = var.prefix 25 | ip_range = var.cidr 26 | 27 | delete_protection = false 28 | } 29 | resource "hcloud_network_subnet" "this" { 30 | count = var.cidr == null ? 0 : 1 31 | network_zone = data.hcloud_location.this.network_zone 32 | network_id = hcloud_network.this[0].id 33 | ip_range = hcloud_network.this[0].ip_range 34 | type = "cloud" 35 | } 36 | 37 | # load balancers 38 | resource "hcloud_load_balancer" "this" { 39 | count = var.load_balancer_ip == null ? 0 : 1 40 | location = data.hcloud_location.this.name 41 | name = var.prefix 42 | load_balancer_type = "lb11" 43 | 44 | delete_protection = false 45 | 46 | algorithm { 47 | type = "round_robin" 48 | } 49 | } 50 | resource "hcloud_load_balancer_network" "this" { 51 | count = var.load_balancer_ip == null ? 0 : 1 52 | subnet_id = hcloud_network_subnet.this[0].id 53 | load_balancer_id = hcloud_load_balancer.this[0].id 54 | ip = var.load_balancer_ip 55 | 56 | enable_public_interface = true 57 | } 58 | -------------------------------------------------------------------------------- /modules/talos-cluster/patches/common.yaml: -------------------------------------------------------------------------------- 1 | cluster: 2 | network: 3 | cni: 4 | name: none 5 | dnsDomain: cluster.local 6 | podSubnets: 7 | - "fc00:1::/96" # ensures the correct ip family is selected, unused otherwise! 8 | # 108 is the largest supported 128b range 9 | # 20 bits is hardcoded here https://github.com/kubernetes/kubernetes/blob/ec16c90aaf5d1f0606747c421c8680bb2b243d4e/cmd/kube-apiserver/app/options/validation.go#L40 10 | serviceSubnets: 11 | - "fc00::/108" 12 | proxy: 13 | disabled: true 14 | machine: 15 | features: 16 | apidCheckExtKeyUsage: true 17 | diskQuotaSupport: true 18 | rbac: true 19 | stableHostname: true 20 | hostDNS: 21 | enabled: true 22 | forwardKubeDNSToHost: false # doesn't work on singlestack ipv6! 169.254.116.108 address is hardcoded! 23 | resolveMemberNames: true 24 | kubePrism: 25 | enabled: true 26 | kubelet: 27 | clusterDNS: 28 | - "fc00::a" 29 | # https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ 30 | extraArgs: 31 | cloud-provider: external 32 | rotate-server-certificates: true 33 | extraConfig: 34 | address: "::" 35 | healthzBindAddress: "::" 36 | network: 37 | kubespan: 38 | enabled: true 39 | advertiseKubernetesNetworks: false 40 | allowDownPeerBypass: false 41 | harvestExtraEndpoints: false 42 | mtu: 1420 43 | filters: 44 | endpoints: 45 | - "::/0" 46 | - "fc00::/108" 47 | - "fc00:1::/96" 48 | sysctls: 49 | net.core.somaxconn: 65535 50 | net.core.netdev_max_backlog: 4096 51 | net.ipv6.conf.all.forwarding: 1 52 | net.ipv4.ip_forward: 1 53 | -------------------------------------------------------------------------------- /modules/hcloud-apply/firewalls.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_firewall" "deny_all" { 2 | name = "${var.pool.prefix}-deny-all" 3 | } 4 | 5 | resource "hcloud_firewall" "this" { 6 | name = var.pool.prefix 7 | 8 | rule { 9 | description = "apiserver" 10 | direction = "in" 11 | protocol = "tcp" 12 | port = "6443" 13 | source_ips = ["::/0"] 14 | } 15 | rule { 16 | description = "talos control planes" 17 | direction = "in" 18 | protocol = "tcp" 19 | port = "50000" 20 | source_ips = ["::/0"] 21 | } 22 | rule { 23 | description = "talos workers" 24 | direction = "in" 25 | protocol = "tcp" 26 | port = "50001" 27 | source_ips = ["::/0"] 28 | } 29 | rule { 30 | description = "https" 31 | direction = "in" 32 | protocol = "tcp" 33 | port = "443" 34 | source_ips = ["::/0"] 35 | } 36 | rule { 37 | description = "http" 38 | direction = "in" 39 | protocol = "tcp" 40 | port = "80" 41 | source_ips = ["::/0"] 42 | } 43 | rule { 44 | description = "healthz" 45 | direction = "in" 46 | protocol = "tcp" 47 | port = "10256" 48 | source_ips = ["::/0"] 49 | } 50 | 51 | # allows full access between cluster nodes 52 | dynamic "rule" { 53 | for_each = toset(["tcp", "udp"]) 54 | content { 55 | direction = "in" 56 | protocol = rule.value 57 | port = "any" 58 | source_ips = [for key, node in var.cluster.nodes : node.public_ip6_network_64] 59 | } 60 | } 61 | dynamic "rule" { 62 | for_each = toset(["icmp", "gre", "esp"]) 63 | content { 64 | direction = "in" 65 | protocol = rule.value 66 | source_ips = [for key, node in var.cluster.nodes : node.public_ip6_network_64] 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /modules/talos-cluster/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | type = string 3 | description = "cluster name" 4 | } 5 | variable "endpoint" { 6 | type = string 7 | description = "cluster endpoint" 8 | 9 | validation { 10 | condition = startswith(var.endpoint, "http") == false && endswith(var.endpoint, "6443") == false 11 | error_message = "must not contain protocol or port" 12 | } 13 | } 14 | variable "talos_version" { 15 | type = string 16 | description = "talos version" 17 | } 18 | variable "kubernetes_version" { 19 | type = string 20 | description = "kubernetes version" 21 | } 22 | 23 | variable "pools" { 24 | type = list(object({ 25 | prefix = string 26 | cidr = optional(string) 27 | control_planes = map(object({ 28 | name = string 29 | aliases = list(string) 30 | public_ip6_network_64 = string 31 | public_ip6_64 = string 32 | public_ip6 = string 33 | patches = list(string) 34 | })) 35 | workers = map(object({ 36 | name = string 37 | aliases = list(string) 38 | public_ip6_network_64 = string 39 | public_ip6_64 = string 40 | public_ip6 = string 41 | patches = list(string) 42 | })) 43 | })) 44 | description = "list of node pool module outputs" 45 | validation { 46 | condition = length([for i, pool in var.pools : pool.prefix]) == length(distinct([for i, pool in var.pools : pool.prefix])) 47 | error_message = "pool prefixes must be unique" 48 | } 49 | validation { 50 | condition = length(compact([for i, pool in var.pools : pool.cidr])) == length(distinct(compact([for i, pool in var.pools : pool.cidr]))) 51 | error_message = "pool cidrs must be unique" 52 | } 53 | } 54 | 55 | variable "patches" { 56 | type = object({ 57 | common = optional(list(string), []) 58 | control_planes = optional(list(string), []) 59 | workers = optional(list(string), []) 60 | }) 61 | description = "cluster-wide config patches" 62 | default = {} 63 | } 64 | -------------------------------------------------------------------------------- /modules/hcloud-pool/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | s1 = { 3 | patches_common = var.cidr == null ? [] : [ 4 | yamlencode({ 5 | machine = { 6 | kubelet = { 7 | nodeIP = { 8 | validSubnets = [ 9 | var.cidr, 10 | ] 11 | } 12 | } 13 | } 14 | }), 15 | ] 16 | } 17 | s2 = { 18 | control_planes = merge([for i, node in var.control_planes : node.removed ? {} : { 19 | "${join("-", [var.prefix, "control-plane", i + 1])}" = merge(node, { 20 | name = join("-", [var.prefix, "control-plane", i + 1]) 21 | private_ip4 = var.cidr == null ? null : cidrhost(var.cidr, i + 11) 22 | patches = flatten([ 23 | local.s1.patches_common, 24 | var.patches.common, 25 | var.patches.control_planes, 26 | node.patches, 27 | ]) 28 | }) 29 | }]...) 30 | workers = merge([for i, node in var.workers : node.removed ? {} : { 31 | "${join("-", [var.prefix, "worker", i + 1])}" = merge(node, { 32 | name = join("-", [var.prefix, "worker", i + 1]) 33 | private_ip4 = var.cidr == null ? null : cidrhost(var.cidr, i + 21) 34 | patches = flatten([ 35 | local.s1.patches_common, 36 | var.patches.common, 37 | var.patches.workers, 38 | node.patches, 39 | ]) 40 | }) 41 | }]...) 42 | } 43 | s3 = { 44 | nodes = merge(local.s2.control_planes, local.s2.workers) 45 | } 46 | s4 = { 47 | ips6 = { for key, ip in hcloud_primary_ip.ips6 : 48 | key => { 49 | public_ip6_id = ip.id 50 | public_ip6_network_64 = ip.ip_network # 2000:2:3:4::/64 51 | public_ip6_64 = "${cidrhost(ip.ip_network, 1)}/64" # 2000:2:3:4::1/64 52 | public_ip6 = cidrhost(ip.ip_network, 1) # 2000:2:3:4::1 53 | } 54 | } 55 | } 56 | 57 | ids = { 58 | network = one(hcloud_network.this[*].id) 59 | subnet = one(hcloud_network_subnet.this[*].id) 60 | load_balancer = one(hcloud_load_balancer.this[*].id) 61 | } 62 | 63 | control_planes = { for key, node in local.s2.control_planes : 64 | key => merge(node, local.s4.ips6[key]) 65 | } 66 | workers = { for key, node in local.s2.workers : 67 | key => merge(node, local.s4.ips6[key]) 68 | } 69 | nodes = merge(local.control_planes, local.workers) 70 | } 71 | -------------------------------------------------------------------------------- /examples/minimal.tf: -------------------------------------------------------------------------------- 1 | data "hcloud_image" "v1_9_5_amd64" { 2 | with_selector = "name=talos,version=v1.9.5,arch=amd64" 3 | } 4 | data "hcloud_datacenter" "nuremberg" { 5 | name = "nbg1-dc3" 6 | } 7 | 8 | locals { 9 | image_id = data.hcloud_image.v1_9_5_amd64.id 10 | } 11 | 12 | module "nuremberg_pool" { 13 | source = "github.com/miran248/terraform-talos-modules//modules/hcloud-pool?ref=v3.2.2" 14 | 15 | prefix = "nbg" 16 | datacenter = data.hcloud_datacenter.nuremberg.name 17 | 18 | control_planes = [ 19 | { server_type = "cx22", image_id = local.image_id }, 20 | ] 21 | workers = [ 22 | { server_type = "cx22", image_id = local.image_id }, 23 | ] 24 | } 25 | 26 | module "talos_cluster" { 27 | source = "github.com/miran248/terraform-talos-modules//modules/talos-cluster?ref=v3.2.2" 28 | 29 | name = "example" 30 | endpoint = "k.example.com" 31 | talos_version = "v1.9.5" 32 | kubernetes_version = "v1.32.2" 33 | 34 | pools = [ 35 | module.nuremberg_pool, 36 | ] 37 | 38 | patches = { 39 | common = [ 40 | yamlencode({ 41 | cluster = { 42 | network = { 43 | cni = { 44 | name = "none" 45 | } 46 | } 47 | inlineManifests = [ 48 | { 49 | name = "hcloud-secret", 50 | contents = <<-EOF 51 | apiVersion: v1 52 | kind: Secret 53 | metadata: 54 | name: hcloud 55 | namespace: kube-system 56 | stringData: 57 | token: ${var.hcloud_token} 58 | type: Opaque 59 | EOF 60 | }, 61 | ] 62 | } 63 | machine = { 64 | network = { 65 | nameservers = [ 66 | "2a00:1098:2b::1", # https://nat64.net 67 | "2a00:1098:2c::1", # https://nat64.net 68 | "2a01:4f8:c2c:123f::1", # https://nat64.net 69 | ] 70 | } 71 | time = { 72 | servers = [ 73 | "/dev/ptp0", 74 | ] 75 | } 76 | } 77 | }), 78 | ] 79 | } 80 | } 81 | 82 | module "hcloud_apply" { 83 | for_each = { for pool in [ 84 | module.nuremberg_pool, 85 | ] : pool.prefix => pool } 86 | source = "github.com/miran248/terraform-talos-modules//modules/hcloud-apply?ref=v3.2.2" 87 | 88 | cluster = module.talos_cluster 89 | pool = each.value 90 | } 91 | 92 | module "talos_apply" { 93 | source = "github.com/miran248/terraform-talos-modules//modules/talos-apply?ref=v3.2.2" 94 | 95 | cluster = module.talos_cluster 96 | } 97 | 98 | resource "google_dns_record_set" "talos_ipv6" { 99 | name = "k.${data.google_dns_managed_zone.this.dns_name}" 100 | managed_zone = data.google_dns_managed_zone.this.name 101 | type = "AAAA" 102 | ttl = 300 103 | 104 | rrdatas = module.talos_cluster.public_ips6.control_planes 105 | } 106 | 107 | # outputs 108 | output "talos_config" { 109 | value = module.talos_cluster.talos_config 110 | sensitive = true 111 | } 112 | -------------------------------------------------------------------------------- /manifests/hcloud-csi/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | helmCharts: 4 | - name: hcloud-csi 5 | repo: https://charts.hetzner.cloud 6 | version: 2.18.2 7 | releaseName: hcloud-csi 8 | namespace: kube-system 9 | includeCRDs: true 10 | valuesInline: 11 | global: 12 | enableProvidedByTopology: true 13 | controller: 14 | replicaCount: 1 15 | hcloudVolumeDefaultLocation: nbg1 16 | priorityClassName: "system-cluster-critical" 17 | resources: 18 | csiAttacher: 19 | limits: 20 | memory: 80Mi 21 | cpu: 50m 22 | requests: 23 | memory: 80Mi 24 | cpu: 50m 25 | csiResizer: 26 | limits: 27 | memory: 80Mi 28 | cpu: 50m 29 | requests: 30 | memory: 80Mi 31 | cpu: 50m 32 | csiProvisioner: 33 | limits: 34 | memory: 80Mi 35 | cpu: 50m 36 | requests: 37 | memory: 80Mi 38 | cpu: 50m 39 | livenessProbe: 40 | limits: 41 | memory: 80Mi 42 | cpu: 50m 43 | requests: 44 | memory: 80Mi 45 | cpu: 50m 46 | hcloudCSIDriver: 47 | limits: 48 | memory: 80Mi 49 | cpu: 100m 50 | requests: 51 | memory: 80Mi 52 | cpu: 100m 53 | affinity: 54 | podAntiAffinity: 55 | requiredDuringSchedulingIgnoredDuringExecution: 56 | - labelSelector: 57 | matchExpressions: 58 | - key: csi-hcloud 59 | operator: In 60 | values: 61 | - controller 62 | topologyKey: "kubernetes.io/hostname" 63 | node: 64 | priorityClassName: "system-node-critical" 65 | resources: 66 | csiNodeDriverRegistrar: 67 | limits: 68 | memory: 40Mi 69 | cpu: 50m 70 | requests: 71 | memory: 40Mi 72 | cpu: 50m 73 | livenessProbe: 74 | limits: 75 | memory: 40Mi 76 | cpu: 50m 77 | requests: 78 | memory: 40Mi 79 | cpu: 50m 80 | hcloudCSIDriver: 81 | limits: 82 | memory: 80Mi 83 | cpu: 100m 84 | requests: 85 | memory: 80Mi 86 | cpu: 100m 87 | hostNetwork: true 88 | # affinity: 89 | # nodeAffinity: 90 | # requiredDuringSchedulingIgnoredDuringExecution: 91 | # nodeSelectorTerms: 92 | # - matchExpressions: 93 | # - key: "node-role.kubernetes.io/control-plane" 94 | # operator: NotIn 95 | # values: 96 | # - "" 97 | # extraEnvVars: 98 | # - name: LOG_LEVEL 99 | # value: trace 100 | # - name: HCLOUD_DEBUG 101 | # value: "1" 102 | metrics: 103 | enabled: true 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # terraform-talos-modules 2 | Modules in this repository help provision and maintain multi-region [kubernetes](https://kubernetes.io) clusters on [hetzner](https://www.hetzner.com). 3 | 4 | ## features 5 | - [talos](https://www.talos.dev) with kubespan, kubeprism and hostdns 6 | - ipv6-only connectivity, with optional ipv4 private networks and load balancers (cilium requires additional configuration) 7 | - single-stack, ipv6 internals (dual-stack possible with additional patches) 8 | - [cilium](https://cilium.io) - direct routing (veth), bigtcp and bbr 9 | - [talos-ccm](https://github.com/siderolabs/talos-cloud-controller-manager) - handles cluster certificates and ipam (CloudAllocator) 10 | - [hcloud-csi](https://github.com/hetznercloud/csi-driver) - storage 11 | - [gcp-wif-webhook](https://github.com/pfnet-research/gcp-workload-identity-federation-webhook) - links kubernetes and gcp service accounts 12 | 13 | ## modules 14 | 1. [hcloud-pool](modules/hcloud-pool) - allocates public ipv6 /64 blocks for all defined nodes and optionally private network and a load balancer 15 | 2. [talos-cluster](modules/talos-cluster) - forms a talos cluster from previously defined node pools 16 | 3. [hcloud-apply](modules/hcloud-apply) - provisions servers with talos configs and sets up firewalls 17 | 4. [talos-apply](modules/talos-apply) - bootstraps the cluster and handles all future config changes 18 | 5. [gcp-wif](modules/gcp-wif) - optional, manages gcp workload identity pool, service accounts and a bucket, it also generates talos config patch 19 | 6. [gcp-wif-apply](modules/gcp-wif-apply) - optional, downloads oidc files from the running cluster and stores them in the previously created bucket 20 | 21 | ## examples 22 | See [examples](examples) folder. 23 | 24 | ## diagram 25 | The following [mermaid](https://github.com/mermaid-js/mermaid) flowchart outlines the order of operations between different modules for a cluster, spanning two regions. 26 | 27 | ```mermaid 28 | %%{init: {'theme': 'neutral' } }%% 29 | graph TD 30 | WIF[/gcp-wif/] 31 | HPN[/hcloud-pool nbg/] 32 | HPH[/hcloud-pool hel/] 33 | TC[talos-cluster] 34 | HAN[hcloud-apply nbg] 35 | HAH[hcloud-apply hel] 36 | TA[talos-apply] 37 | WIFA[gcp-wif-apply] 38 | HPN --> TC 39 | HPN --> HAN 40 | HPH --> TC 41 | TC --> HAN 42 | TC --> TA 43 | TC --> HAH 44 | HPH --> HAH 45 | TC --> WIFA 46 | TA --> WIFA 47 | WIF --> WIFA 48 | WIF --> TC 49 | ``` 50 | 51 | ## try it out 52 | 1. clone the repo 53 | 2. navigate to [dev](dev) folder and run [just](https://github.com/casey/just) to deploy the cluster (run it again, when it fails, to apply the gcp-wif) 54 | 3. navigate to [base](/) folder, open talos dashboard and wait for `[talos] created` messages 55 | ```shell 56 | > TALOSCONFIG=talos-config talosctl -n c1 dashboard 57 | ``` 58 | 4. run `just` to generate all necessary yaml files 59 | 5. apply them individually, `talos-ccm` and `cilium` are required 60 | ```shell 61 | > KUBECONFIG=kube-config kubectl apply --server-side=true -f .build/manifests/talos-ccm.yaml 62 | > KUBECONFIG=kube-config kubectl apply --server-side=true -f .build/manifests/cilium.yaml 63 | ... 64 | ``` 65 | 6. open talos dashboard again and wait for the message `[talos] machine is running and ready`. 66 | 7. to verify, open [k9s](https://k9scli.io/) 67 | ```shell 68 | > KUBECONFIG=kube-config k9s 69 | ``` 70 | -------------------------------------------------------------------------------- /dev/1-talos.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | dev1_image_id = data.hcloud_image.v1_10_6_amd64.id 3 | } 4 | 5 | module "dev1_nuremberg_pool" { 6 | source = "../modules/hcloud-pool" 7 | 8 | prefix = "dev1-nbg" 9 | datacenter = data.hcloud_datacenter.nuremberg.name 10 | 11 | # cidr = "192.168.1.0/24" 12 | # load_balancer_ip = "192.168.1.5" 13 | 14 | control_planes = [ 15 | { server_type = "cx22", image_id = local.dev1_image_id }, 16 | { server_type = "cx22", image_id = local.dev1_image_id }, 17 | { server_type = "cx22", image_id = local.dev1_image_id }, 18 | ] 19 | workers = [ 20 | { server_type = "cx22", image_id = local.dev1_image_id }, 21 | { server_type = "cx22", image_id = local.dev1_image_id }, 22 | { server_type = "cx22", image_id = local.dev1_image_id }, 23 | ] 24 | } 25 | 26 | # module "dev1_helsinki_pool" { 27 | # source = "../modules/hcloud-pool" 28 | 29 | # prefix = "dev1-hel" 30 | # datacenter = data.hcloud_datacenter.helsinki.name 31 | 32 | # workers = [ 33 | # { server_type = "cx22", image_id = local.dev1_image_id }, 34 | # # { server_type = "cx22", image_id = local.dev1_image_id }, 35 | # ] 36 | # } 37 | 38 | locals { 39 | dev1_pools = [ 40 | module.dev1_nuremberg_pool, 41 | # module.dev1_helsinki_pool, 42 | ] 43 | } 44 | 45 | module "dev1_talos_cluster" { 46 | source = "../modules/talos-cluster" 47 | 48 | name = "dev1" 49 | endpoint = "dev1.dev.248.sh" 50 | talos_version = "v1.10.6" 51 | kubernetes_version = "v1.33.3" 52 | 53 | pools = local.dev1_pools 54 | 55 | patches = { 56 | common = [ 57 | <<-EOF 58 | cluster: 59 | network: 60 | cni: 61 | name: none 62 | machine: 63 | network: 64 | nameservers: 65 | - 2a00:1098:2b::1 # https://nat64.net 66 | - 2a00:1098:2c::1 # https://nat64.net 67 | - 2a01:4f8:c2c:123f::1 # https://nat64.net 68 | time: 69 | servers: 70 | - /dev/ptp0 71 | EOF 72 | ] 73 | control_planes = flatten([ 74 | module.gcp_wif.patches.control_planes, 75 | # <<-EOF 76 | # cluster: 77 | # allowSchedulingOnControlPlanes: true 78 | # EOF 79 | ]) 80 | } 81 | } 82 | 83 | module "dev1_hcloud_apply" { 84 | for_each = { for pool in local.dev1_pools : pool.prefix => pool } 85 | source = "../modules/hcloud-apply" 86 | 87 | cluster = module.dev1_talos_cluster 88 | pool = each.value 89 | } 90 | 91 | module "dev1_talos_apply" { 92 | source = "../modules/talos-apply" 93 | 94 | cluster = module.dev1_talos_cluster 95 | } 96 | 97 | module "dev1_gcp_wif_apply" { 98 | source = "../modules/gcp-wif-apply" 99 | 100 | identities = module.gcp_wif 101 | cluster = module.dev1_talos_cluster 102 | apply = module.dev1_talos_apply 103 | } 104 | 105 | resource "google_dns_record_set" "dev1_talos_ipv6" { 106 | name = "${module.dev1_talos_cluster.name}.${data.google_dns_managed_zone.this.dns_name}" 107 | managed_zone = data.google_dns_managed_zone.this.name 108 | type = "AAAA" 109 | ttl = 300 110 | 111 | rrdatas = module.dev1_talos_cluster.public_ips6.control_planes 112 | } 113 | 114 | # outputs 115 | output "talos_config" { 116 | value = module.dev1_talos_cluster.talos_config 117 | sensitive = true 118 | } 119 | -------------------------------------------------------------------------------- /modules/talos-cluster/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | s1 = { 3 | control_planes = merge([for i, pool in var.pools : pool.control_planes]...) 4 | workers = merge([for i, pool in var.pools : pool.workers]...) 5 | } 6 | s2 = { 7 | nodes = merge(local.s1.control_planes, local.s1.workers) 8 | 9 | aliases = { 10 | control_planes = { for key, node in local.s1.control_planes : 11 | key => flatten([node.aliases, "c${index(keys(local.s1.control_planes), key) + 1}"]) 12 | } 13 | workers = { for key, node in local.s1.workers : 14 | key => flatten([node.aliases, "w${index(keys(local.s1.workers), key) + 1}"]) 15 | } 16 | } 17 | } 18 | s3 = { 19 | aliases = merge(local.s2.aliases.control_planes, local.s2.aliases.workers) 20 | } 21 | s4 = { 22 | cert_sans = flatten([ 23 | var.endpoint, 24 | [for key, node in local.s2.nodes : [ 25 | node.public_ip6, 26 | local.s3.aliases[key], 27 | ]], 28 | ]) 29 | } 30 | s5 = { 31 | patches_common = flatten([ 32 | file("${path.module}/patches/common.yaml"), 33 | yamlencode({ 34 | machine = { 35 | certSANs = local.s4.cert_sans 36 | network = { 37 | extraHostEntries = [for key, node in local.s2.nodes : { 38 | ip = node.public_ip6 39 | aliases = local.s3.aliases[key] 40 | }] 41 | } 42 | } 43 | }), 44 | var.patches.common, 45 | ]) 46 | } 47 | s6 = { 48 | patches = { 49 | control_planes = flatten([ 50 | local.s5.patches_common, 51 | file("${path.module}/patches/control-planes.yaml"), 52 | yamlencode({ 53 | cluster = { 54 | apiServer = { 55 | certSANs = local.s4.cert_sans 56 | } 57 | etcd = { 58 | advertisedSubnets = [for key, node in local.s2.nodes : node.public_ip6_network_64] 59 | } 60 | } 61 | }), 62 | var.patches.control_planes, 63 | ]) 64 | workers = flatten([ 65 | local.s5.patches_common, 66 | var.patches.workers, 67 | ]) 68 | } 69 | } 70 | s7 = { 71 | control_planes = { for key, node in local.s1.control_planes : 72 | key => merge(node, { 73 | talos = { machine_type = "controlplane" } 74 | patches = flatten([ 75 | local.s6.patches.control_planes, 76 | node.patches, 77 | ]) 78 | }) 79 | } 80 | workers = { for key, node in local.s1.workers : 81 | key => merge(node, { 82 | talos = { machine_type = "worker" } 83 | patches = flatten([ 84 | local.s6.patches.workers, 85 | node.patches, 86 | ]) 87 | }) 88 | } 89 | } 90 | 91 | cluster_endpoint = "https://${var.endpoint}:6443" 92 | 93 | nodes = merge(local.s7.control_planes, local.s7.workers) 94 | 95 | names = { 96 | control_planes = [for key, node in local.s1.control_planes : node.name] 97 | workers = [for key, node in local.s1.workers : node.name] 98 | } 99 | public_ips6 = { 100 | control_planes = [for key, node in local.s1.control_planes : node.public_ip6] 101 | workers = [for key, node in local.s1.workers : node.public_ip6] 102 | } 103 | 104 | configs = { for key, config in data.talos_machine_configuration.this : 105 | key => yamlencode(yamldecode(config.machine_configuration)) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /examples/multi-region.tf: -------------------------------------------------------------------------------- 1 | data "hcloud_image" "v1_9_5_amd64" { 2 | with_selector = "name=talos,version=v1.9.5,arch=amd64" 3 | } 4 | data "hcloud_datacenter" "nuremberg" { 5 | name = "nbg1-dc3" 6 | } 7 | data "hcloud_datacenter" "helsinki" { 8 | name = "hel1-dc2" 9 | } 10 | 11 | locals { 12 | patches_zitadel = <<-EOF 13 | machine: 14 | nodeLabels: 15 | app: zitadel 16 | EOF 17 | 18 | image_id = data.hcloud_image.v1_9_5_amd64.id 19 | } 20 | 21 | module "nuremberg_pool" { 22 | source = "github.com/miran248/terraform-talos-modules//modules/hcloud-pool?ref=v3.2.2" 23 | 24 | prefix = "nbg" 25 | datacenter = data.hcloud_datacenter.nuremberg.name 26 | 27 | control_planes = [ 28 | { server_type = "cx22", image_id = local.image_id }, 29 | { server_type = "cx22", image_id = local.image_id }, 30 | { server_type = "cx22", image_id = local.image_id }, 31 | ] 32 | workers = [ 33 | { server_type = "cx22", image_id = local.image_id, patches = [local.patches_zitadel] }, 34 | # { server_type = "cx22", image_id = local.image_id, patches = [local.patches_zitadel] }, 35 | # { server_type = "cx22", image_id = local.image_id, patches = [local.patches_zitadel] }, 36 | ] 37 | } 38 | 39 | module "helsinki_pool" { 40 | source = "github.com/miran248/terraform-talos-modules//modules/hcloud-pool?ref=v3.2.2" 41 | 42 | prefix = "hel" 43 | datacenter = data.hcloud_datacenter.helsinki.name 44 | 45 | workers = [ 46 | { server_type = "cx22", image_id = local.image_id }, 47 | # { server_type = "cx22", image_id = local.image_id }, 48 | ] 49 | } 50 | 51 | module "talos_cluster" { 52 | source = "github.com/miran248/terraform-talos-modules//modules/talos-cluster?ref=v3.2.2" 53 | 54 | name = "example" 55 | endpoint = "k.example.com" 56 | talos_version = "v1.9.5" 57 | kubernetes_version = "v1.32.2" 58 | 59 | pools = [ 60 | module.nuremberg_pool, 61 | module.helsinki_pool, 62 | ] 63 | 64 | patches = { 65 | common = [ 66 | yamlencode({ 67 | cluster = { 68 | network = { 69 | cni = { 70 | name = "none" 71 | } 72 | } 73 | inlineManifests = [ 74 | { 75 | name = "hcloud-secret", 76 | contents = <<-EOF 77 | apiVersion: v1 78 | kind: Secret 79 | metadata: 80 | name: hcloud 81 | namespace: kube-system 82 | stringData: 83 | token: ${var.hcloud_token} 84 | type: Opaque 85 | EOF 86 | }, 87 | ] 88 | } 89 | machine = { 90 | network = { 91 | nameservers = [ 92 | "2a00:1098:2b::1", # https://nat64.net 93 | "2a00:1098:2c::1", # https://nat64.net 94 | "2a01:4f8:c2c:123f::1", # https://nat64.net 95 | ] 96 | } 97 | time = { 98 | servers = [ 99 | "/dev/ptp0", 100 | ] 101 | } 102 | } 103 | }), 104 | ] 105 | } 106 | } 107 | 108 | module "hcloud_apply" { 109 | for_each = { for pool in [ 110 | module.nuremberg_pool, 111 | module.helsinki_pool, 112 | ] : pool.prefix => pool } 113 | source = "github.com/miran248/terraform-talos-modules//modules/hcloud-apply?ref=v3.2.2" 114 | 115 | cluster = module.talos_cluster 116 | pool = each.value 117 | } 118 | 119 | module "talos_apply" { 120 | source = "github.com/miran248/terraform-talos-modules//modules/talos-apply?ref=v3.2.2" 121 | 122 | cluster = module.talos_cluster 123 | } 124 | 125 | resource "google_dns_record_set" "talos_ipv6" { 126 | name = "k.${data.google_dns_managed_zone.this.dns_name}" 127 | managed_zone = data.google_dns_managed_zone.this.name 128 | type = "AAAA" 129 | ttl = 300 130 | 131 | rrdatas = module.talos_cluster.public_ips6.control_planes 132 | } 133 | 134 | # outputs 135 | output "talos_config" { 136 | value = module.talos_cluster.talos_config 137 | sensitive = true 138 | } 139 | -------------------------------------------------------------------------------- /manifests/cilium/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.3.0/experimental-install.yaml 5 | helmCharts: 6 | - name: cilium 7 | repo: https://helm.cilium.io 8 | version: v1.18.0 9 | releaseName: cilium 10 | namespace: kube-system 11 | includeCRDs: true 12 | apiVersions: 13 | - gateway.networking.k8s.io/v1/GatewayClass 14 | # - monitoring.coreos.com/v1 15 | valuesInline: 16 | # https://docs.cilium.io/en/stable/helm-reference/ 17 | defs: 18 | hubble: &hubble false 19 | ipv6: &ipv6 true 20 | ipv4: &ipv4 false 21 | rollout: &rollout true 22 | serviceMonitor: &serviceMonitor 23 | serviceMonitor: 24 | enabled: false 25 | interval: 60s 26 | 27 | # debug: 28 | # enabled: true 29 | 30 | # cni: 31 | # uninstall: true 32 | 33 | # cleanState: true 34 | # cleanBpfState: true 35 | 36 | # TODO: try tunnel again 37 | routingMode: native 38 | 39 | k8sServiceHost: localhost 40 | k8sServicePort: 7445 41 | 42 | ipam: 43 | mode: kubernetes 44 | 45 | kubeProxyReplacement: true 46 | 47 | ipv6: 48 | enabled: *ipv6 49 | ipv4: 50 | enabled: *ipv4 51 | 52 | k8s: 53 | requireIPv6PodCIDR: *ipv6 54 | requireIPv4PodCIDR: *ipv4 55 | 56 | enableIPv6Masquerade: false 57 | enableIPv4Masquerade: false 58 | 59 | enableIPv6BIGTCP: *ipv6 60 | enableIPv4BIGTCP: *ipv4 61 | 62 | autoDirectNodeRoutes: false 63 | directRoutingSkipUnreachable: false 64 | 65 | localRedirectPolicy: false 66 | 67 | underlayProtocol: ipv6 68 | 69 | hostFirewall: 70 | enabled: false 71 | 72 | endpointRoutes: 73 | enabled: false 74 | 75 | externalIPs: 76 | enabled: true 77 | 78 | nodePort: 79 | enabled: true 80 | bindProtection: false 81 | 82 | hostPort: 83 | enabled: true 84 | 85 | l7Proxy: true 86 | 87 | bgp: 88 | enabled: false 89 | announce: 90 | loadbalancerIP: false 91 | podCIDR: false 92 | 93 | bgpControlPlane: 94 | enabled: false 95 | 96 | nat46x64Gateway: 97 | enabled: false 98 | 99 | nodeIPAM: 100 | enabled: false 101 | 102 | bpf: 103 | # datapathMode: veth 104 | datapathMode: netkit 105 | masquerade: false 106 | tproxy: true 107 | hostLegacyRouting: false 108 | distributedLRU: 109 | enabled: true 110 | mapDynamicSizeRatio: 0.08 111 | 112 | bpfClockProbe: true 113 | 114 | bandwidthManager: 115 | enabled: true 116 | bbr: true 117 | 118 | # native causes the following error 119 | # interface kubespan: attaching program cil_xdp_entry using bpf_link: create link: operation not supported 120 | # loadBalancer: 121 | # acceleration: native 122 | loadBalancer: 123 | acceleration: best-effort 124 | 125 | authentication: 126 | enabled: false 127 | mutual: 128 | spire: 129 | enabled: false 130 | 131 | encryption: 132 | enabled: false 133 | type: wireguard 134 | nodeEncryption: false # broken 135 | 136 | gatewayAPI: 137 | enabled: true 138 | hostNetwork: 139 | enabled: true 140 | 141 | socketLB: 142 | enabled: true 143 | 144 | kubeProxyReplacementHealthzBindAddr: "[::]:10256" 145 | 146 | rollOutCiliumPods: *rollout 147 | 148 | operator: 149 | replicas: 1 150 | rollOutPods: *rollout 151 | prometheus: 152 | enabled: true 153 | <<: *serviceMonitor 154 | 155 | hubble: 156 | enabled: *hubble 157 | enableOpenMetrics: true 158 | preferIpv6: *ipv6 159 | metrics: 160 | enabled: 161 | - dns 162 | - drop 163 | - tcp 164 | - flow 165 | - icmp 166 | - port-distribution 167 | - http 168 | <<: *serviceMonitor 169 | relay: 170 | enabled: *hubble 171 | rollOutPods: *rollout 172 | prometheus: 173 | <<: *serviceMonitor 174 | ui: 175 | enabled: *hubble 176 | rollOutPods: *rollout 177 | service: 178 | annotations: 179 | omni-kube-service-exposer.sidero.dev/port: "50080" 180 | omni-kube-service-exposer.sidero.dev/label: Hubble 181 | frontend: 182 | server: 183 | ipv6: 184 | enabled: *ipv6 185 | envoy: 186 | enabled: true 187 | rollOutPods: *rollout 188 | prometheus: 189 | <<: *serviceMonitor 190 | securityContext: 191 | capabilities: 192 | keepCapNetBindService: true 193 | envoy: 194 | - NET_BIND_SERVICE 195 | - NET_ADMIN 196 | - SYS_ADMIN 197 | - BPF 198 | 199 | prometheus: 200 | enabled: true 201 | <<: *serviceMonitor 202 | 203 | securityContext: 204 | privileged: true 205 | capabilities: 206 | ciliumAgent: 207 | - CHOWN 208 | - KILL 209 | - NET_ADMIN 210 | - NET_RAW 211 | - IPC_LOCK 212 | - SYS_ADMIN 213 | - SYS_RESOURCE 214 | - DAC_OVERRIDE 215 | - FOWNER 216 | - SETGID 217 | - SETUID 218 | cleanCiliumState: 219 | - NET_ADMIN 220 | - SYS_ADMIN 221 | - SYS_RESOURCE 222 | cgroup: 223 | autoMount: 224 | enabled: false 225 | hostRoot: /sys/fs/cgroup 226 | # patches: 227 | # - path: patch-namespaces.yaml 228 | -------------------------------------------------------------------------------- /dev/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/devops-rob/terracurl" { 5 | version = "2.1.0" 6 | hashes = [ 7 | "h1:RY+mLsFOrgzoEEle4uROwI6c3HVSyTTcK2lW0ZFPOVg=", 8 | "zh:1ae8588d83a3ee3f270788207643fb435f4bb7cbf194edfa8c5c1628c2bf7d76", 9 | "zh:3bcc038010c63a00867d467494d328f821887b5c44e2eaaf54144184c4251ee1", 10 | "zh:73eafa4d6aefc2fed5d430bfd779bfadc03722d68d65ffb276079696c4bc5529", 11 | "zh:9752cd3280abb509be029c6190dc812fb43efa2f8c398ff30afcc27c3c8974da", 12 | "zh:a49f5ab9f44dd1f3d94a8bd56c62c91074fd186f4a78d6f4655f27c9c5308751", 13 | "zh:d31e4f4a473fde202efe426791b34d8342f6b1b984daaa48887b440ee6af96e8", 14 | ] 15 | } 16 | 17 | provider "registry.terraform.io/hashicorp/google" { 18 | version = "6.48.0" 19 | hashes = [ 20 | "h1:ylB0JzWIfA905LY6Djor7C1FRCvJJqQ+413IIrw/Y/o=", 21 | "zh:3640317ada2141ab5b54b5c5c2b1d0f60af861fb9788f05808b5d5d7ac8d1819", 22 | "zh:4ec6082c343d7ea097bf725c7f550978789860f43b442a7709e1a4ca209d10a7", 23 | "zh:66dcb85612f82c464a682e03a20e9dfd479e743d63a7e9cf754dcf81fa6e442b", 24 | "zh:88ce6cb6f2f5f3a356c27ca5923594d272ea25801ed81026e102a2dc24d347d3", 25 | "zh:b62e3715bf35fe0322b2a56c5072ecfdea2dd25f1c794f432acd1dd0105112b5", 26 | "zh:bc2a9397c6d9b71ccf74cea442e74d8bb5603d491aa4a7cd88d5c44693674e16", 27 | "zh:c6710acbdbaeca8307467186b0cc5184c8223fd0efa8c52e7f81310f34c68c69", 28 | "zh:ca077af69ab66444bf027b7663e1386f2d14699b50562d1674c84522316bf52a", 29 | "zh:e1b74bcdacb20d73da0470f6874dc7ac38b58bfa738ec5e337fcc7c5a4f32663", 30 | "zh:e9b2dcd583025b69b7e80c9ad0d9ee4ed91e731f2073db5b6516978cf7617017", 31 | "zh:f0d4ef4a8ca749b9102b152aafc093fa7def2afbda771b6b26cbc873f4e6dfc8", 32 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 33 | ] 34 | } 35 | 36 | provider "registry.terraform.io/hashicorp/local" { 37 | version = "2.5.3" 38 | hashes = [ 39 | "h1:MCzg+hs1/ZQ32u56VzJMWP9ONRQPAAqAjuHuzbyshvI=", 40 | "zh:284d4b5b572eacd456e605e94372f740f6de27b71b4e1fd49b63745d8ecd4927", 41 | "zh:40d9dfc9c549e406b5aab73c023aa485633c1b6b730c933d7bcc2fa67fd1ae6e", 42 | "zh:6243509bb208656eb9dc17d3c525c89acdd27f08def427a0dce22d5db90a4c8b", 43 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 44 | "zh:885d85869f927853b6fe330e235cd03c337ac3b933b0d9ae827ec32fa1fdcdbf", 45 | "zh:bab66af51039bdfcccf85b25fe562cbba2f54f6b3812202f4873ade834ec201d", 46 | "zh:c505ff1bf9442a889ac7dca3ac05a8ee6f852e0118dd9a61796a2f6ff4837f09", 47 | "zh:d36c0b5770841ddb6eaf0499ba3de48e5d4fc99f4829b6ab66b0fab59b1aaf4f", 48 | "zh:ddb6a407c7f3ec63efb4dad5f948b54f7f4434ee1a2607a49680d494b1776fe1", 49 | "zh:e0dafdd4500bec23d3ff221e3a9b60621c5273e5df867bc59ef6b7e41f5c91f6", 50 | "zh:ece8742fd2882a8fc9d6efd20e2590010d43db386b920b2a9c220cfecc18de47", 51 | "zh:f4c6b3eb8f39105004cf720e202f04f57e3578441cfb76ca27611139bc116a82", 52 | ] 53 | } 54 | 55 | provider "registry.terraform.io/hashicorp/tls" { 56 | version = "4.1.0" 57 | hashes = [ 58 | "h1:zEv9tY1KR5vaLSyp2lkrucNJ+Vq3c+sTFK9GyQGLtFs=", 59 | "zh:14c35d89307988c835a7f8e26f1b83ce771e5f9b41e407f86a644c0152089ac2", 60 | "zh:2fb9fe7a8b5afdbd3e903acb6776ef1be3f2e587fb236a8c60f11a9fa165faa8", 61 | "zh:35808142ef850c0c60dd93dc06b95c747720ed2c40c89031781165f0c2baa2fc", 62 | "zh:35b5dc95bc75f0b3b9c5ce54d4d7600c1ebc96fbb8dfca174536e8bf103c8cdc", 63 | "zh:38aa27c6a6c98f1712aa5cc30011884dc4b128b4073a4a27883374bfa3ec9fac", 64 | "zh:51fb247e3a2e88f0047cb97bb9df7c228254a3b3021c5534e4563b4007e6f882", 65 | "zh:62b981ce491e38d892ba6364d1d0cdaadcee37cc218590e07b310b1dfa34be2d", 66 | "zh:bc8e47efc611924a79f947ce072a9ad698f311d4a60d0b4dfff6758c912b7298", 67 | "zh:c149508bd131765d1bc085c75a870abb314ff5a6d7f5ac1035a8892d686b6297", 68 | "zh:d38d40783503d278b63858978d40e07ac48123a2925e1a6b47e62179c046f87a", 69 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 70 | "zh:fb07f708e3316615f6d218cec198504984c0ce7000b9f1eebff7516e384f4b54", 71 | ] 72 | } 73 | 74 | provider "registry.terraform.io/hetznercloud/hcloud" { 75 | version = "1.52.0" 76 | hashes = [ 77 | "h1:8Juiz/B0XWpSCJmIYLBoGqU14R0W9rudwVInfd7jBt0=", 78 | "zh:1e9bb6b6a2ea5f441638dbae2d60fbe04ff455f58a18c740b8b7913e2197d875", 79 | "zh:29c122e404ba331cfbadacc7f1294de5a31c9dfd60bdfe3e1b402271fc8e419c", 80 | "zh:2bd0ae2f0bb9f16b7753f59a08e57ac7230f9c471278d7882f81406b9426c8c7", 81 | "zh:4383206971873f6b5d81580a9a36e0158924f5816ebb6206b0cf2430e4e6a609", 82 | "zh:47e2ca1cfa18500e4952ab51dc357a0450d00a92da9ea03e452f1f3efe6bbf75", 83 | "zh:8e9fe90e3cea29bb7892b64da737642fc22b0106402df76c228a3cbe99663278", 84 | "zh:a2d69350a69c471ddb63bcc74e105e585319a0fc0f4d1b7f70569f6d2ece5824", 85 | "zh:a97abcc254e21c294e2d6b0fc9068acfd63614b097dda365f1c56ea8b0fd5f6b", 86 | "zh:aba8d72d4fe2e89c922d5446d329e5c23d00b28227b4666e6486ba18ea2ec278", 87 | "zh:ad36c333978c2d9e4bc43dcadcbff42fe771a8c5ef53d028bcacec8287bf78a7", 88 | "zh:cdb1e6903b9d2f0ad8845d4eb390fbe724ee2435fb045baeab38d4319e637682", 89 | "zh:df77b08757f3f36b8aadb33d73362320174047044414325c56a87983f48b5186", 90 | "zh:e07513d5ad387247092b5ae1c87e21a387fc51873b3f38eee616187e38b090a7", 91 | "zh:e2be02bdc59343ff4b9e26c3b93db7680aaf3e6ed13c8c4c4b144c74c2689915", 92 | ] 93 | } 94 | 95 | provider "registry.terraform.io/siderolabs/talos" { 96 | version = "0.8.1" 97 | hashes = [ 98 | "h1:fWms1ojEGxaCDSQ5ezjO2MdpVx0cEIAKNCmkB2vKuUw=", 99 | "zh:02aeea4001ea216d37fd948e0760971f2525d31609d75dd1a7871f483e43260d", 100 | "zh:0bd6d2f9b6daf9cec0e20d1e22cad635983b5c071c106a3bec51be283c9fa254", 101 | "zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d", 102 | "zh:290ced18cfa372681d53522b5ea6c392206e90181ef0884719768b3ef627d077", 103 | "zh:3270a27a483d2be332915e339b910a4810fb16505e060fc4a988b0d653f06d90", 104 | "zh:34f91c967ae25219abc81d21a477d3fc514c62a73084bfb9d3d2d1490e98070b", 105 | "zh:46a0eb4397e97d9dc354087ce1b16ccdca3876b0e339d7ec1919002a43aa0a6d", 106 | "zh:4d382770c97675c6b4355a91f50b38f9b6bd088707834c9efb308e608bbdae48", 107 | "zh:6ec2828c419615cce850ba1eafffee2797cae62876999f3b0a163c17f579c97a", 108 | "zh:9405b011c631d9fb001d8b96a1657e071181434960543e857fb14ec2230618b0", 109 | "zh:a9a6f6824793e811ec52c0b1c4b8d19855c851120a91f7617ffa7e36aa65710a", 110 | "zh:aa472818c7880c7cf19f5bd584dcf513a3daa2fe636d4af730b0bcf495eadebf", 111 | "zh:cd6037f6267987fb606f98b1a425d71a1826289ac39e62973a45b60f0f37de06", 112 | "zh:ddea6372fef17de6648018c4c64b87acecaba9f5443fcf46ff3d92c048605b30", 113 | "zh:e367b0359c8b413f705ded1d0d7b4a3c09cee1bd0028337faa80a150b08b945a", 114 | ] 115 | } 116 | --------------------------------------------------------------------------------