├── .github └── workflows │ └── terraform.yml ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── HA_Proxy.md └── Variables.md ├── image.png ├── main.tf ├── modules ├── domain │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── proxy │ ├── main.tf │ ├── output.tf │ └── variables.tf ├── nginx-example ├── ingress.yaml.example └── nginx-controller.yaml ├── scripts ├── docker.sh ├── haproxy.sh ├── ip.sh ├── template.sh └── versions.sh ├── templates ├── haproxy.tmpl └── talosctl.tmpl ├── terraform.tfvars.example └── variables.tf /.github/workflows/terraform.yml: -------------------------------------------------------------------------------- 1 | name: 'Terraform' 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | terraform: 13 | name: 'Terraform' 14 | runs-on: ubuntu-latest 15 | environment: production 16 | 17 | permissions: 18 | # Give the default GITHUB_TOKEN write permission to commit and push the 19 | # added or changed files to the repository. 20 | contents: write 21 | 22 | # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest 23 | defaults: 24 | run: 25 | shell: bash 26 | 27 | steps: 28 | 29 | # Checkout the repository to the GitHub Actions runner 30 | - name: Checkout 31 | uses: actions/checkout@v3 32 | 33 | # Install terraform 34 | - name: Setup Terraform 35 | uses: hashicorp/setup-terraform@v3 36 | with: 37 | terraform_version: "^1.3.7" 38 | terraform_wrapper: false 39 | 40 | # Copy the terraform.tfvars.example file for variables 41 | - name: Create terraform.tfvars 42 | run: cp ./terraform.tfvars.example ./terraform.tfvars 43 | 44 | # Create random SSH keys 45 | - name: Create random SSH keys 46 | run: mkdir ~/.ssh && touch ~/.ssh/id_rsa && touch ~/.ssh/id_rsa.pub 47 | 48 | # Initialize Terraform working directory by creating initial files, loading any remote state, downloading modules, etc. 49 | - name: Terraform Init 50 | run: terraform init 51 | 52 | # Validate Terraform files 53 | - name: Terraform Validate 54 | run: terraform validate 55 | 56 | # Format Terraform files 57 | - name: Terraform Format 58 | run: terraform fmt --recursive 59 | 60 | # Commit files 61 | - name: Commit and Push 62 | uses: stefanzweifel/git-auto-commit-action@v5.0.0 63 | with: 64 | commit_message: 'Formatted terraform files' 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | .terraform.lock.hcl 8 | 9 | # Crash log files 10 | crash.log 11 | crash.*.log 12 | 13 | # Exclude all .tfvars files, which are likely to contain sensitive data, such as 14 | # password, private keys, and other secrets. These should not be part of version 15 | # control as they are data points which are potentially sensitive and subject 16 | # to change depending on the environment. 17 | *.tfvars 18 | *.tfvars.json 19 | 20 | # Ignore override files as they are usually used to override resources locally and so 21 | # are not checked in 22 | override.tf 23 | override.tf.json 24 | *_override.tf 25 | *_override.tf.json 26 | 27 | # Include override files you do wish to add to version control using negated pattern 28 | # !example_override.tf 29 | 30 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 31 | # example: *tfplan* 32 | 33 | # Ignore CLI configuration files 34 | .terraformrc 35 | terraform.rc 36 | 37 | # Custom project paths 38 | output/* 39 | terraform.tfvars 40 | controlplane.yaml 41 | haproxy.cfg 42 | talos_setup.sh 43 | talosconfig 44 | worker.yaml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Naman Arora 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # talos-proxmox-cluster 2 | 3 | [![Terraform](https://github.com/Naman1997/talos-proxmox-cluster/actions/workflows/terraform.yml/badge.svg)](https://github.com/Naman1997/talos-proxmox-cluster/actions/workflows/terraform.yml) 4 | [![GitHub license](https://img.shields.io/github/license/Naereen/StrapDown.js.svg)](https://github.com/Naman1997/talos-proxmox-cluster/blob/main/LICENSE) 5 | 6 | Automated talos cluster with system extensions 7 | 8 | ## Dependencies 9 | 10 | | Dependency | Location | 11 | | ------ | ------ | 12 | | [Proxmox](https://www.proxmox.com/en/proxmox-ve) | Proxmox node | 13 | | [xz](https://en.wikipedia.org/wiki/XZ_Utils) | Proxmox node | 14 | | [jq](https://stedolan.github.io/jq/) | Client | 15 | | [arp-scan](https://linux.die.net/man/1/arp-scan) | Client | 16 | | [talosctl](https://www.talos.dev/latest/learn-more/talosctl/) | Client | 17 | | [OpenTofu](https://opentofu.org/) | Client | 18 | | [HAproxy](http://www.haproxy.org/) | Raspberry Pi | 19 | | [Docker](https://docs.docker.com/) | Client | 20 | 21 | `Client` refers to the node that will be executing `tofu apply` to create the cluster. The `Raspberry Pi` can be replaced with a VM or a LXC container. 22 | 23 | Docker is mandatory on the `Client` as this projects builds a custom talos image with system extensions using the [imager](https://github.com/siderolabs/talos/pkgs/container/installer) docker image on the `Client` itself. 24 | 25 | ## Options for creation of HA Proxy Server 26 | 27 | The `main` banch will automatically create a VM for a load balancer with 2 CPUs and 2 GiB of memory on your Proxmox node. 28 | 29 | You can use the [no-lb](https://github.com/Naman1997/simple-talos-cluster/tree/no-lb) branch in case you do not want to use an external load-balancer. This branch uses the 1st master node that gets created as the cluster endpoint. 30 | 31 | Another option is to use the [manual-lb](https://github.com/Naman1997/simple-talos-cluster/tree/manual-lb) branch in case you wish to create an external lb manually. 32 | 33 | ## Create the terraform.tfvars file 34 | 35 | The variables needed to configure this script are documented in this [doc](docs/Variables.md). 36 | 37 | ``` 38 | cp terraform.tfvars.example terraform.tfvars 39 | # Edit and save the variables according to your liking 40 | vim terraform.tfvars 41 | ``` 42 | 43 | ## Enable the Snippets feature in Proxmox 44 | 45 | In the proxmox web portal, go to `Datacenter` > `Storage` > Click on `local` > `Edit` > Under `Content` choose `Snippets` > Click on `OK` to save. 46 | 47 | ![local directory](image.png) 48 | 49 | 50 | ## Create the cluster 51 | 52 | ``` 53 | tofu init -upgrade 54 | tofu plan 55 | tofu apply --auto-approve 56 | ``` 57 | 58 | ## Expose your cluster to the internet (Optional) 59 | 60 | It is possible to expose your cluster to the internet over a small vps even if both your vps and your public ips are dynamic. This is possible by setting up dynamic dns for both your internal network and the vps using something like duckdns 61 | and a docker container to regularly monitor the IP addresses on both ends. A connection can be then made using wireguard to traverse the network between these 2 nodes. This way you can hide your public IP while exposing services to the internet. 62 | 63 | Project Link: [wireguard-k8s-lb](https://github.com/Naman1997/wireguard-k8s-lb) 64 | 65 | 66 | ## Known Issue(s) 67 | 68 | ### Proxmox in KVM 69 | 70 | Currently this only happens if you're running this inside on a proxmox node that itself is virtualized inside kvm. This is highly unlikely, but I'll make a note of this for anyone stuck on this. 71 | 72 | This project uses `arp-scan` to scan the local network using arp requests. In case your user does not have proper permissions to scan using the `virbr0` interface, then the talos VMs will not be found. 73 | 74 | To fix this, either you can give your user access to the interface by adding it to `libvirt`, `libvirt-qemu` and `kvm` groups or you can just use `sudo`, in case you opt for solution 2, make sure to run the `talosctl kubeconfig` command generated for you in `talos_setup.sh` after `tofu apply` finishes. -------------------------------------------------------------------------------- /docs/HA_Proxy.md: -------------------------------------------------------------------------------- 1 | # HA Proxy user setup 2 | 3 | It's a good idea to create a non-root user just to manage haproxy access. In this example, the user is named `wireproxy`. 4 | 5 | ``` 6 | # Login to the Raspberry Pi 7 | # Install haproxy 8 | sudo apt-get install haproxy 9 | sudo systemctl enable haproxy 10 | sudo systemctl start haproxy 11 | # Run this from a user with sudo privileges 12 | sudo EDITOR=vim visudo 13 | %wireproxy ALL= (root) NOPASSWD: /bin/systemctl restart haproxy 14 | 15 | sudo addgroup wireproxy 16 | sudo adduser --disabled-password --ingroup wireproxy wireproxy 17 | ``` 18 | 19 | You'll need to make sure that you're able to ssh into this user account without a password. For example, let's say the user with sudo privileges is named `ubuntu`. Follow these steps to enable passwordless SSH for `ubuntu`. 20 | 21 | ``` 22 | # Run this from your Client 23 | # Change user/IP address here as needed 24 | ssh-copy-id -i ~/.ssh/id_rsa.pub ubuntu@192.168.0.100 25 | ``` 26 | 27 | Now you can either follow the same steps for the `wireproxy` user (not recommended as we don't want to give the `wireproxy` user a password) or you can copy the `~/.ssh/authorized_keys` file from the `ubuntu` user to this user. 28 | 29 | ``` 30 | # Login to the Raspberry Pi with user 'ubuntu' 31 | cat ~/.ssh/authorized_keys 32 | # Copy the value in a clipboard 33 | sudo su wireproxy 34 | # You're now logged in as wireproxy user 35 | mkdir -p ~/.ssh 36 | vim ~/.ssh/authorized_keys 37 | # Paste the same key here 38 | # Logout from the Raspberry Pi 39 | # Make sure you're able to ssh in wireproxy user from your Client 40 | ssh wireproxy@192.168.0.100 41 | ``` 42 | 43 | Using the same example, the user `wireproxy` needs to own the files under `/etc/haproxy` 44 | 45 | ``` 46 | # Login to the Raspberry Pi with user 'ubuntu' 47 | sudo chown -R wireproxy: /etc/haproxy 48 | ``` 49 | -------------------------------------------------------------------------------- /docs/Variables.md: -------------------------------------------------------------------------------- 1 | # Variables needed for terraform.tfvars file 2 | 3 | | Variable | Description | 4 | | ------ | ------ | 5 | | system_type | System Type. Valid values: intel / amd | 6 | | PROXMOX_API_ENDPOINT | API endpoint for proxmox | 7 | | PROXMOX_USERNAME | User name used to login proxmox | 8 | | PROXMOX_PASSWORD | Password used to login proxmox | 9 | | PROXMOX_IP | IP address for proxmox | 10 | | DEFAULT_BRIDGE | Bridge to use when creating VMs in proxmox | 11 | | TARGET_NODE | Target node name in proxmox | 12 | | SSH_KEY | Path to SSH key to be used for copying the talos image and creating a template | 13 | | cluster_name | Cluster name to be used for kubeconfig | 14 | | MASTER_COUNT | Number of masters to create | 15 | | WORKER_COUNT | Number of workers to create | 16 | | autostart | Enable/Disable VM start on host bootup | 17 | | master_config | Kubernetes master config | 18 | | worker_config | Kubernetes worker config | 19 | | ha_proxy_server | IP address of server running haproxy | 20 | | ha_proxy_user | User on ha_proxy_server that can modify '/etc/haproxy/haproxy.cfg' and restart haproxy.service | 21 | | ha_proxy_server | SSH key used to log in ha_proxy_server | -------------------------------------------------------------------------------- /image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Naman1997/simple-talos-cluster/49f7e6e2ef8ecae6dbb91a9782c3d5d3522672bf/image.png -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | 2 | terraform { 3 | required_providers { 4 | docker = { 5 | source = "kreuzwerker/docker" 6 | version = "3.0.2" 7 | } 8 | proxmox = { 9 | source = "bpg/proxmox" 10 | version = "0.75.0" 11 | } 12 | } 13 | } 14 | 15 | provider "proxmox" { 16 | endpoint = var.PROXMOX_API_ENDPOINT 17 | username = "${var.PROXMOX_USERNAME}@pam" 18 | password = var.PROXMOX_PASSWORD 19 | insecure = true 20 | } 21 | 22 | data "external" "versions" { 23 | program = ["${path.module}/scripts/versions.sh"] 24 | } 25 | 26 | locals { 27 | ha_proxy_user = "ubuntu" 28 | qemu_ga_version = data.external.versions.result["qemu_ga_version"] 29 | amd_ucode_version = data.external.versions.result["amd_ucode_version"] 30 | intel_ucode_version = data.external.versions.result["intel_ucode_version"] 31 | imager_version = data.external.versions.result["imager_version"] 32 | system_command = var.system_type == "amd" ? [ 33 | "metal", 34 | "--system-extension-image", 35 | "ghcr.io/siderolabs/qemu-guest-agent:${local.qemu_ga_version}", 36 | "--system-extension-image", 37 | "ghcr.io/siderolabs/intel-ucode:${local.amd_ucode_version}" 38 | ] : [ 39 | "metal", 40 | "--system-extension-image", 41 | "ghcr.io/siderolabs/qemu-guest-agent:${local.qemu_ga_version}", 42 | "--system-extension-image", 43 | "ghcr.io/siderolabs/intel-ucode:${local.intel_ucode_version}" 44 | ] 45 | } 46 | 47 | provider "docker" {} 48 | 49 | resource "docker_image" "imager" { 50 | name = "ghcr.io/siderolabs/imager:${local.imager_version}" 51 | } 52 | 53 | resource "null_resource" "cleanup" { 54 | provisioner "local-exec" { 55 | command = "mkdir -p output && rm -f talos_setup.sh haproxy.cfg talosconfig worker.yaml controlplane.yaml" 56 | working_dir = path.root 57 | } 58 | } 59 | 60 | resource "docker_container" "imager" { 61 | 62 | depends_on = [ 63 | null_resource.cleanup, 64 | data.external.versions 65 | ] 66 | 67 | image = docker_image.imager.image_id 68 | name = "imager" 69 | privileged = true 70 | tty = true 71 | rm = true 72 | attach = false 73 | command = local.system_command 74 | volumes { 75 | container_path = "/dev" 76 | host_path = "/dev" 77 | } 78 | volumes { 79 | container_path = "/out" 80 | host_path = "${abspath(path.module)}/output" 81 | } 82 | } 83 | 84 | resource "null_resource" "wait_for_imager" { 85 | depends_on = [docker_container.imager] 86 | provisioner "local-exec" { 87 | command = "/bin/bash scripts/docker.sh" 88 | } 89 | } 90 | 91 | resource "null_resource" "copy_image" { 92 | depends_on = [null_resource.wait_for_imager] 93 | provisioner "remote-exec" { 94 | connection { 95 | host = var.PROXMOX_IP 96 | user = var.PROXMOX_USERNAME 97 | private_key = file(var.SSH_KEY) 98 | } 99 | 100 | inline = [ 101 | "rm -rf /root/talos", 102 | "mkdir /root/talos" 103 | ] 104 | } 105 | 106 | provisioner "file" { 107 | source = "${path.root}/output/metal-amd64.raw.zst" 108 | destination = "/root/talos/talos.raw.zst" 109 | connection { 110 | type = "ssh" 111 | host = var.PROXMOX_IP 112 | user = var.PROXMOX_USERNAME 113 | private_key = file(var.SSH_KEY) 114 | } 115 | } 116 | } 117 | 118 | resource "null_resource" "create_template" { 119 | depends_on = [null_resource.copy_image] 120 | provisioner "remote-exec" { 121 | when = create 122 | connection { 123 | host = var.PROXMOX_IP 124 | user = var.PROXMOX_USERNAME 125 | private_key = file(var.SSH_KEY) 126 | } 127 | script = "${path.root}/scripts/template.sh" 128 | } 129 | } 130 | 131 | module "master_domain" { 132 | 133 | depends_on = [null_resource.create_template] 134 | 135 | source = "./modules/domain" 136 | count = var.MASTER_COUNT 137 | name = format("talos-master-%s", count.index) 138 | memory = var.master_config.memory 139 | vcpus = var.master_config.vcpus 140 | sockets = var.master_config.sockets 141 | autostart = var.autostart 142 | default_bridge = var.DEFAULT_BRIDGE 143 | target_node = var.TARGET_NODE 144 | scan_interface = var.INTERFACE_TO_SCAN 145 | } 146 | 147 | module "worker_domain" { 148 | 149 | depends_on = [null_resource.create_template] 150 | 151 | source = "./modules/domain" 152 | count = var.WORKER_COUNT 153 | name = format("talos-worker-%s", count.index) 154 | memory = var.worker_config.memory 155 | vcpus = var.worker_config.vcpus 156 | sockets = var.worker_config.sockets 157 | autostart = var.autostart 158 | default_bridge = var.DEFAULT_BRIDGE 159 | target_node = var.TARGET_NODE 160 | scan_interface = var.INTERFACE_TO_SCAN 161 | } 162 | 163 | module "proxy" { 164 | source = "./modules/proxy" 165 | ha_proxy_user = local.ha_proxy_user 166 | DEFAULT_BRIDGE = var.DEFAULT_BRIDGE 167 | TARGET_NODE = var.TARGET_NODE 168 | ssh_key = join("", [var.SSH_KEY, ".pub"]) 169 | } 170 | 171 | resource "local_file" "haproxy_config" { 172 | depends_on = [ 173 | module.master_domain.node, 174 | module.worker_domain.node, 175 | module.proxy.node 176 | ] 177 | content = templatefile("${path.root}/templates/haproxy.tmpl", 178 | { 179 | node_map_masters = zipmap( 180 | tolist(module.master_domain.*.address), tolist(module.master_domain.*.name) 181 | ), 182 | node_map_workers = zipmap( 183 | tolist(module.worker_domain.*.address), tolist(module.worker_domain.*.name) 184 | ) 185 | } 186 | ) 187 | filename = "haproxy.cfg" 188 | 189 | provisioner "file" { 190 | source = "${path.root}/haproxy.cfg" 191 | destination = "/etc/haproxy/haproxy.cfg" 192 | connection { 193 | type = "ssh" 194 | host = module.proxy.proxy_ipv4_address 195 | user = local.ha_proxy_user 196 | private_key = file(var.SSH_KEY) 197 | } 198 | } 199 | 200 | provisioner "remote-exec" { 201 | connection { 202 | host = module.proxy.proxy_ipv4_address 203 | user = local.ha_proxy_user 204 | private_key = file(var.SSH_KEY) 205 | } 206 | script = "${path.root}/scripts/haproxy.sh" 207 | } 208 | } 209 | 210 | resource "local_file" "talosctl_config" { 211 | depends_on = [ 212 | module.master_domain.node, 213 | module.worker_domain.node, 214 | module.proxy.node, 215 | resource.local_file.haproxy_config 216 | ] 217 | content = templatefile("${path.root}/templates/talosctl.tmpl", 218 | { 219 | load_balancer = module.proxy.proxy_ipv4_address, 220 | node_map_masters = tolist(module.master_domain.*.address), 221 | node_map_workers = tolist(module.worker_domain.*.address) 222 | primary_controller = module.master_domain[0].address 223 | } 224 | ) 225 | filename = "talos_setup.sh" 226 | file_permission = "755" 227 | } 228 | 229 | resource "null_resource" "create_cluster" { 230 | depends_on = [local_file.talosctl_config] 231 | provisioner "local-exec" { 232 | command = "/bin/bash talos_setup.sh" 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /modules/domain/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | proxmox = { 4 | source = "bpg/proxmox" 5 | version = "0.75.0" 6 | } 7 | } 8 | } 9 | 10 | resource "proxmox_virtual_environment_vm" "node" { 11 | name = var.name 12 | on_boot = var.autostart 13 | node_name = var.target_node 14 | bios = "ovmf" 15 | scsi_hardware = "virtio-scsi-pci" 16 | timeout_shutdown_vm = 300 17 | 18 | memory { 19 | dedicated = var.memory 20 | } 21 | 22 | cpu { 23 | cores = var.vcpus 24 | type = "x86-64-v2" 25 | sockets = var.sockets 26 | } 27 | 28 | agent { 29 | enabled = true 30 | timeout = "10s" 31 | } 32 | 33 | clone { 34 | retries = 3 35 | vm_id = 8000 36 | full = true 37 | } 38 | 39 | network_device { 40 | model = "virtio" 41 | bridge = var.default_bridge 42 | } 43 | } 44 | 45 | data "external" "address" { 46 | depends_on = [proxmox_virtual_environment_vm.node] 47 | working_dir = path.root 48 | program = ["bash", "scripts/ip.sh", "${lower(proxmox_virtual_environment_vm.node.network_device[0].mac_address)}", "${var.scan_interface}"] 49 | } 50 | -------------------------------------------------------------------------------- /modules/domain/outputs.tf: -------------------------------------------------------------------------------- 1 | output "address" { 2 | value = data.external.address.result["address"] 3 | description = "IP Address of the node" 4 | } 5 | 6 | output "name" { 7 | value = proxmox_virtual_environment_vm.node.name 8 | description = "Name of the node" 9 | } -------------------------------------------------------------------------------- /modules/domain/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | description = "Name of node" 3 | type = string 4 | } 5 | 6 | variable "memory" { 7 | description = "Amount of memory needed" 8 | type = string 9 | } 10 | 11 | variable "vcpus" { 12 | description = "Number of vcpus" 13 | type = number 14 | } 15 | 16 | variable "sockets" { 17 | description = "Number of sockets" 18 | type = number 19 | } 20 | 21 | variable "autostart" { 22 | description = "Enable/Disable VM start on host bootup" 23 | type = bool 24 | } 25 | 26 | variable "default_bridge" { 27 | description = "Bridge to use when creating VMs in proxmox" 28 | type = string 29 | } 30 | 31 | variable "target_node" { 32 | description = "Target node name in proxmox" 33 | type = string 34 | } 35 | 36 | variable "scan_interface" { 37 | description = "Interface that you wish to scan for finding the talos VMs" 38 | type = string 39 | } -------------------------------------------------------------------------------- /modules/proxy/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | proxmox = { 4 | source = "bpg/proxmox" 5 | version = "0.75.0" 6 | } 7 | } 8 | } 9 | 10 | data "local_file" "ssh_public_key" { 11 | filename = pathexpand(var.ssh_key) 12 | } 13 | 14 | resource "proxmox_virtual_environment_file" "cloud_config" { 15 | content_type = "snippets" 16 | datastore_id = "local" 17 | node_name = var.TARGET_NODE 18 | 19 | source_raw { 20 | data = <<-EOF 21 | #cloud-config 22 | users: 23 | - default 24 | - name: ${var.ha_proxy_user} 25 | groups: 26 | - sudo 27 | shell: /bin/bash 28 | ssh_authorized_keys: 29 | - ${trimspace(data.local_file.ssh_public_key.content)} 30 | sudo: ALL=(ALL) NOPASSWD:ALL 31 | runcmd: 32 | - apt update -y && apt dist-upgrade -y 33 | - apt install -y qemu-guest-agent haproxy net-tools unattended-upgrades 34 | - timedatectl set-timezone America/Toronto 35 | - systemctl enable qemu-guest-agent 36 | - systemctl enable --now haproxy 37 | - systemctl start qemu-guest-agent 38 | - chown -R ${var.ha_proxy_user}:${var.ha_proxy_user} /etc/haproxy/ 39 | - echo "done" > /tmp/cloud-config.done 40 | EOF 41 | 42 | file_name = "cloud-config.yaml" 43 | } 44 | } 45 | 46 | resource "proxmox_virtual_environment_download_file" "ubuntu_cloud_image" { 47 | content_type = "iso" 48 | datastore_id = "local" 49 | node_name = var.TARGET_NODE 50 | url = "https://cloud-images.ubuntu.com/oracular/current/oracular-server-cloudimg-amd64.img" 51 | upload_timeout = 1000 52 | overwrite = false 53 | } 54 | 55 | resource "proxmox_virtual_environment_vm" "node" { 56 | name = "haproxy" 57 | node_name = var.TARGET_NODE 58 | 59 | agent { 60 | enabled = true 61 | } 62 | 63 | cpu { 64 | cores = 2 65 | } 66 | 67 | memory { 68 | dedicated = 2048 69 | } 70 | 71 | disk { 72 | datastore_id = "local-lvm" 73 | file_id = proxmox_virtual_environment_download_file.ubuntu_cloud_image.id 74 | interface = "virtio0" 75 | iothread = true 76 | discard = "on" 77 | size = 20 78 | } 79 | 80 | initialization { 81 | ip_config { 82 | ipv4 { 83 | address = "dhcp" 84 | } 85 | } 86 | 87 | user_data_file_id = proxmox_virtual_environment_file.cloud_config.id 88 | } 89 | 90 | network_device { 91 | bridge = var.DEFAULT_BRIDGE 92 | } 93 | 94 | provisioner "local-exec" { 95 | command = <<-EOT 96 | n=0 97 | until [ "$n" -ge 10 ] 98 | do 99 | echo "Attempt number: $n" 100 | ssh-keygen -R $ADDRESS 101 | if [ $? -eq 0 ]; then 102 | echo "Successfully removed $ADDRESS" 103 | break 104 | fi 105 | n=$((n+1)) 106 | sleep $[ ( $RANDOM % 10 ) + 1 ]s 107 | done 108 | EOT 109 | environment = { 110 | ADDRESS = element(flatten(self.ipv4_addresses), 1) 111 | } 112 | when = destroy 113 | } 114 | 115 | provisioner "local-exec" { 116 | command = <<-EOT 117 | n=0 118 | until [ "$n" -ge 10 ] 119 | do 120 | echo "Attempt number: $n" 121 | ssh-keyscan -H $ADDRESS >> ~/.ssh/known_hosts 122 | ssh -q -o StrictHostKeyChecking=no ${var.ha_proxy_user}@$ADDRESS exit < /dev/null 123 | if [ $? -eq 0 ]; then 124 | echo "Successfully added $ADDRESS" 125 | break 126 | fi 127 | n=$((n+1)) 128 | sleep $[ ( $RANDOM % 10 ) + 1 ]s 129 | done 130 | EOT 131 | environment = { 132 | ADDRESS = element(flatten(self.ipv4_addresses), 1) 133 | } 134 | when = create 135 | } 136 | 137 | } -------------------------------------------------------------------------------- /modules/proxy/output.tf: -------------------------------------------------------------------------------- 1 | output "proxy_ipv4_address" { 2 | value = proxmox_virtual_environment_vm.node.ipv4_addresses[1][0] 3 | } -------------------------------------------------------------------------------- /modules/proxy/variables.tf: -------------------------------------------------------------------------------- 1 | variable "ha_proxy_user" { 2 | description = "Username for proxy VM" 3 | type = string 4 | } 5 | 6 | variable "DEFAULT_BRIDGE" { 7 | description = "Bridge to use when creating VMs in proxmox" 8 | type = string 9 | } 10 | 11 | variable "TARGET_NODE" { 12 | description = "Target node name in proxmox" 13 | type = string 14 | } 15 | 16 | variable "ssh_key" { 17 | description = "Public SSH key to be authorized" 18 | } -------------------------------------------------------------------------------- /nginx-example/ingress.yaml.example: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1 2 | kind: Ingress 3 | metadata: 4 | name: minimal-ingress 5 | spec: 6 | ingressClassName: nginx 7 | rules: 8 | - host: example.duckdns.org 9 | http: 10 | paths: 11 | - path: / 12 | pathType: Prefix 13 | backend: 14 | service: 15 | name: nginx 16 | port: 17 | number: 80 -------------------------------------------------------------------------------- /nginx-example/nginx-controller.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | hostPort: 3 | enabled: true 4 | ports: 5 | http: 80 6 | https: 443 7 | 8 | kind: DaemonSet 9 | service: 10 | # Change your Load Balancer's IP here 11 | externalIPs: [192.168.0.101] -------------------------------------------------------------------------------- /scripts/docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | IMAGE_NAME=imager 3 | MAX_RETRIES=30 4 | RETRY_INTERVAL=5 5 | 6 | for ((i = 1; i <= MAX_RETRIES; i++)); do 7 | container_count=$(docker ps --filter "NAME=$IMAGE_NAME" | grep $IMAGE_NAME | wc -l) 8 | if [[ "$container_count" -eq 0 ]]; then 9 | sleep 5 10 | exit 0 11 | fi 12 | 13 | if [ $i -lt $MAX_RETRIES ]; then 14 | sleep $RETRY_INTERVAL 15 | else 16 | echo "Maximum retries reached. Address not found." 17 | exit 1 18 | fi 19 | done -------------------------------------------------------------------------------- /scripts/haproxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | n=0 3 | retries=5 4 | 5 | until [ "$n" -ge "$retries" ]; do 6 | if sudo systemctl restart haproxy; then 7 | exit 0 8 | else 9 | n=$((n+1)) 10 | sleep 5 11 | fi 12 | done 13 | 14 | echo "All retries failed. Exiting with code 1." 15 | exit 1 16 | -------------------------------------------------------------------------------- /scripts/ip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MAX_RETRIES=30 4 | RETRY_INTERVAL=5 5 | IP_REGEX='^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$' 6 | 7 | 8 | for ((i = 1; i <= MAX_RETRIES; i++)); do 9 | if [ -n "$2" ]; then 10 | address=$(arp-scan --localnet -I "$2" | grep "$1" | awk '{ printf $1 }') 11 | else 12 | address=$(arp-scan --localnet | grep "$1" | awk '{ printf $1 }') 13 | fi 14 | 15 | 16 | if [[ -n "$address" && "$address" =~ $IP_REGEX ]]; then 17 | jq -n --arg address "$address" '{"address":$address}' 18 | exit 0 19 | fi 20 | 21 | if [ $i -lt $MAX_RETRIES ]; then 22 | sleep $RETRY_INTERVAL 23 | else 24 | echo "Maximum retries reached. Address not found." 25 | exit 1 26 | fi 27 | done -------------------------------------------------------------------------------- /scripts/template.sh: -------------------------------------------------------------------------------- 1 | BOOTDISK=scsi0 2 | qm destroy 8000 || true 3 | zstd --decompress talos/talos.raw.zst -o talos/talos.raw 4 | sleep 3 5 | qm create 8000 --memory 2048 --net0 virtio,bridge=vmbr0 --agent 1 --cores 2 --sockets 1 --cpu cputype=x86-64-v2 6 | qm importdisk 8000 talos/talos.raw local-lvm 7 | qm set 8000 --scsihw virtio-scsi-pci --scsi0 local-lvm:vm-8000-disk-0,cache=writeback,discard=on 8 | qm set 8000 --boot c --bootdisk $BOOTDISK 9 | qm resize 8000 $BOOTDISK +20G 10 | qm set 8000 --ipconfig0 ip=dhcp 11 | qm set 8000 --bios ovmf 12 | qm set 8000 -efidisk0 local-lvm:0,format=raw,efitype=4m,pre-enrolled-keys=0 13 | qm set 8000 --name talos-golden --template 1 14 | 15 | # Make sure vmid exists 16 | sleep 10 17 | while ! qm config 8000 >/dev/null 2>&1; do 18 | sleep 5 19 | done 20 | 21 | # Make sure disk resize happened 22 | qm=$(qm config 8000 | grep "$BOOTDISK" | cut -d "," -f 4 | cut -d "=" -f 2 | sed s/"M"// | tail -1) 23 | while [ "$qm" -lt 20000 ]; do 24 | qm=$(qm config 8000 | grep "$BOOTDISK" | cut -d "," -f 4 | cut -d "=" -f 2 | sed s/"M"// | tail -1) 25 | sleep 5 26 | done -------------------------------------------------------------------------------- /scripts/versions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | list_versions() { 4 | repository="$1" 5 | token=$(curl -s "https://ghcr.io/token?scope=repository:$repository:pull" | jq -r '.token') 6 | last_version="" 7 | all_versions=() 8 | 9 | while true; do 10 | response=$(curl -s -H "Authorization: Bearer $token" "https://ghcr.io/v2/$repository/tags/list?n=1000&last=$last_version") 11 | tags=$(echo "$response" | jq -r '.tags') 12 | if [ "$tags" == "null" ]; then 13 | break 14 | fi 15 | 16 | readarray -t versions < <(echo "$response" | jq -c '.tags[]' | sed 's/"//g') 17 | for version in "${versions[@]}"; do 18 | all_versions+=("$version") 19 | last_version="$version" 20 | done 21 | done 22 | 23 | for version in "${all_versions[@]}"; do 24 | echo "$version" 25 | done 26 | } 27 | 28 | # Get latest version of intel-ucode 29 | intel_ucode_version=0 30 | for element in $(list_versions "siderolabs/intel-ucode"); do 31 | if [[ "$element" =~ ^[0-9]+$ && "$element" -gt "$intel_ucode_version" ]]; then 32 | intel_ucode_version="$element" 33 | fi 34 | done 35 | 36 | if [[ "$intel_ucode_version" -eq 0 ]]; then 37 | echo "Unable to find the latest version for siderolabs/intel-ucode." 38 | exit 1 39 | fi 40 | 41 | # echo "The latest version of siderolabs/intel-ucode is: $intel_ucode_version" 42 | 43 | # Get latest version of amd-ucode 44 | amd_ucode_version=0 45 | for element in $(list_versions "siderolabs/amd-ucode"); do 46 | if [[ "$element" =~ ^[0-9]+$ && "$element" -gt "$amd_ucode_version" ]]; then 47 | amd_ucode_version="$element" 48 | fi 49 | done 50 | 51 | if [[ "$amd_ucode_version" -eq 0 ]]; then 52 | echo "Unable to find the latest version for siderolabs/amd-ucode." 53 | exit 1 54 | fi 55 | 56 | # echo "The latest version of siderolabs/intel-ucode is: $amd_ucode_version" 57 | 58 | # Get latest version of qemu-guest-agent 59 | qemu_ga_version="" 60 | largest_major=0 61 | largest_minor=0 62 | largest_patch=0 63 | 64 | # Loop through the bash array and find the largest version 65 | for version in $(list_versions "siderolabs/qemu-guest-agent"); do 66 | if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 67 | major="${version%%.*}" 68 | minor="${version#*.}" 69 | minor="${minor%%.*}" 70 | patch="${version##*.}" 71 | 72 | if [[ "$major" -gt "$largest_major" || ("$major" -eq "$largest_major" && "$minor" -gt "$largest_minor") || ("$major" -eq "$largest_major" && "$minor" -eq "$largest_minor" && "$patch" -gt "$largest_patch") ]]; then 73 | largest_major="$major" 74 | largest_minor="$minor" 75 | largest_patch="$patch" 76 | qemu_ga_version="$version" 77 | fi 78 | fi 79 | done 80 | 81 | if [[ -z "$qemu_ga_version" ]]; then 82 | echo "Unable to find the latest version for siderolabs/qemu-guest-agent." 83 | exit 1 84 | fi 85 | 86 | # echo "The latest version of siderolabs/qemu-guest-agent is: $qemu_ga_version" 87 | 88 | imager_version="" 89 | for version in $(list_versions "siderolabs/imager"); do 90 | if [[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 91 | imager_version="$version" 92 | fi 93 | done 94 | 95 | if [[ -z "$imager_version" ]]; then 96 | echo "Unable to find the latest version for siderolabs/imager." 97 | exit 1 98 | fi 99 | 100 | # echo "The latest version of siderolabs/imager is: $imager_version" 101 | 102 | jq -n --arg intel_ucode_version "$intel_ucode_version"\ 103 | --arg amd_ucode_version "$amd_ucode_version" \ 104 | --arg qemu_ga_version "$qemu_ga_version" \ 105 | --arg imager_version "$imager_version" \ 106 | '{"intel_ucode_version":$intel_ucode_version, "amd_ucode_version":$amd_ucode_version, "qemu_ga_version":$qemu_ga_version, "imager_version":$imager_version}' -------------------------------------------------------------------------------- /templates/haproxy.tmpl: -------------------------------------------------------------------------------- 1 | global 2 | log /dev/log local0 3 | log /dev/log local1 notice 4 | chroot /var/lib/haproxy 5 | stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners 6 | stats timeout 30s 7 | user haproxy 8 | group haproxy 9 | daemon 10 | 11 | # Default SSL material locations 12 | ca-base /etc/ssl/certs 13 | crt-base /etc/ssl/private 14 | 15 | # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate 16 | ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 17 | ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 18 | ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets 19 | 20 | defaults 21 | timeout connect 5s 22 | timeout client 1m 23 | timeout server 1m 24 | 25 | frontend kubeAPI 26 | bind :6443 27 | mode tcp 28 | default_backend kubeAPI_backend 29 | frontend talosctl 30 | bind :50000 31 | mode tcp 32 | default_backend talosctl_backend 33 | frontend controller 34 | bind :50001 35 | mode tcp 36 | default_backend controller_backend 37 | 38 | backend kubeAPI_backend 39 | mode tcp 40 | %{ for node_host, node_hostname in node_map_masters ~} 41 | server ${node_hostname} ${node_host}:6443 check check-ssl verify none 42 | %{endfor} 43 | backend talosctl_backend 44 | mode tcp 45 | %{ for node_host, node_hostname in node_map_masters ~} 46 | server ${node_hostname} ${node_host}:50000 check check-ssl verify none 47 | %{endfor} 48 | backend controller_backend 49 | mode tcp 50 | %{ for node_host, node_hostname in node_map_masters ~} 51 | server ${node_hostname} ${node_host}:50001 check check-ssl verify none 52 | %{endfor} 53 | 54 | listen stats 55 | bind *:9000 56 | mode http 57 | stats enable 58 | stats uri / 59 | 60 | -------------------------------------------------------------------------------- /templates/talosctl.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | talosctl machineconfig gen mycluster https://${load_balancer}:6443 3 | sleep 3 4 | %{ for node_host in node_map_masters ~} 5 | talosctl apply-config --insecure --nodes ${node_host} --file controlplane.yaml 6 | echo "Applied controller config to ${node_host}" 7 | %{endfor} 8 | %{ for node_host in node_map_workers ~} 9 | talosctl apply-config --insecure --nodes ${node_host} --file worker.yaml 10 | echo "Applied worker config to ${node_host}" 11 | %{endfor} 12 | 13 | # Bootstrap 14 | sleep 30 15 | talosctl bootstrap --nodes ${primary_controller} -e ${primary_controller} --talosconfig=./talosconfig 16 | echo "Started bootstrap process" 17 | sleep 30 18 | 19 | # Health check 20 | n=0 21 | retries=5 22 | until [ "$n" -ge "$retries" ]; do 23 | if talosctl --talosconfig=./talosconfig --nodes ${primary_controller} -e ${primary_controller} health; then 24 | break 25 | else 26 | n=$((n+1)) 27 | sleep 5 28 | fi 29 | done 30 | 31 | # Update kubeconfig 32 | talosctl kubeconfig --nodes ${primary_controller} -e ${primary_controller} --talosconfig=./talosconfig --force 33 | echo "Updated kubeconfig" 34 | echo "Successfully created cluster" -------------------------------------------------------------------------------- /terraform.tfvars.example: -------------------------------------------------------------------------------- 1 | # Valid values are intel/amd 2 | system_type="intel" 3 | 4 | # Hypervisor config 5 | # Make sure `ssh PROXMOX_USERNAME@ -i ` works 6 | PROXMOX_API_ENDPOINT = "https://192.168.0.103:8006/api2/json" 7 | PROXMOX_USERNAME = "root" 8 | PROXMOX_PASSWORD = "password" 9 | PROXMOX_IP = "192.168.0.100" 10 | DEFAULT_BRIDGE = "vmbr0" 11 | TARGET_NODE = "pve" 12 | SSH_KEY = "/home/user/.ssh/id_rsa" 13 | 14 | # Cluster config 15 | cluster_name = "talos-cluster-1" 16 | MASTER_COUNT = 1 17 | WORKER_COUNT = 1 18 | autostart = true 19 | master_config = { 20 | memory = "2048" 21 | vcpus = 2 22 | sockets = 1 23 | } 24 | worker_config = { 25 | memory = "2048" 26 | vcpus = 2 27 | sockets = 1 28 | } 29 | 30 | # Leave this empty if you are not sure/have a single NIC 31 | # Change this to virbr0 if you're running proxmox inside a KVM VM 32 | # You may need to use sudo for terrafrom apply due to this 33 | INTERFACE_TO_SCAN="" -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "system_type" { 2 | description = "System type" 3 | type = string 4 | 5 | validation { 6 | condition = var.system_type == "intel" || var.system_type == "amd" 7 | error_message = "Valid values for system_type are 'intel' or 'amd'" 8 | } 9 | } 10 | 11 | # Hypervisor config 12 | variable "PROXMOX_API_ENDPOINT" { 13 | description = "API endpoint for proxmox" 14 | type = string 15 | } 16 | 17 | variable "PROXMOX_USERNAME" { 18 | description = "User name used to login proxmox" 19 | type = string 20 | } 21 | 22 | variable "PROXMOX_PASSWORD" { 23 | description = "Password used to login proxmox" 24 | type = string 25 | } 26 | 27 | variable "PROXMOX_IP" { 28 | description = "IP address for proxmox" 29 | type = string 30 | } 31 | 32 | variable "DEFAULT_BRIDGE" { 33 | description = "Bridge to use when creating VMs in proxmox" 34 | type = string 35 | } 36 | 37 | variable "TARGET_NODE" { 38 | description = "Target node name in proxmox" 39 | type = string 40 | } 41 | 42 | variable "SSH_KEY" { 43 | description = "Path to SSH key to be used for copying the talos image and creating a template" 44 | type = string 45 | } 46 | 47 | # Cluster config 48 | variable "cluster_name" { 49 | description = "Cluster name to be used for kubeconfig" 50 | type = string 51 | } 52 | 53 | variable "MASTER_COUNT" { 54 | description = "Number of masters to create" 55 | type = number 56 | validation { 57 | condition = var.MASTER_COUNT != 0 58 | error_message = "Number of master nodes cannot be 0" 59 | } 60 | } 61 | 62 | variable "WORKER_COUNT" { 63 | description = "Number of workers to create" 64 | type = number 65 | } 66 | 67 | variable "autostart" { 68 | description = "Enable/Disable VM start on host bootup" 69 | type = bool 70 | } 71 | 72 | variable "master_config" { 73 | description = "Kubernetes master config" 74 | type = object({ 75 | memory = string 76 | vcpus = number 77 | sockets = number 78 | }) 79 | } 80 | 81 | variable "worker_config" { 82 | description = "Kubernetes worker config" 83 | type = object({ 84 | memory = string 85 | vcpus = number 86 | sockets = number 87 | }) 88 | } 89 | 90 | variable "INTERFACE_TO_SCAN" { 91 | description = "Interface that you wish to scan for finding the talos VMs. Leave this empty for default value." 92 | type = string 93 | } --------------------------------------------------------------------------------