├── .gitignore
├── README.md
├── ansible.cfg
├── create-k8s-vm.yaml
├── group_vars
    └── example-all.yaml
└── inventories
    └── proxmox.py


/.gitignore:
--------------------------------------------------------------------------------
1 | group_vars/all.yaml
2 | mycluster/
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # PROXMOX KUBERNETES BOOTSTRAP
  2 | 
  3 | ## Description
  4 | This is a collection of resources to get a Kubernetes cluster up and running in a Proxmox Virtual Environment. These tools and commands assume the user is executing in a Linux or Linux-like environment.
  5 | # Prerequisites
  6 | 
  7 | * Proxmox 8 or newer server up and running
  8 | * Install Python 3 (Comes pre-installed on most Linux distros)
  9 | * [Generate a set of SSH keys](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#generating-a-new-ssh-key)
 10 | * [Install Docker](https://github.com/docker/docker-install#usage)
 11 | * [Install Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html)
 12 | 
 13 |     ```bash
 14 |     # There are many ways to install Ansible. pipx is my prefered method.
 15 |     python3 -m pip install pipx
 16 |     pipx install --include-deps ansible
 17 |     ```
 18 | 
 19 |     Note: in some environments you may need to use `python` instead of `python3`
 20 | 
 21 | # Create Kubenetes nodes
 22 | 
 23 | ## Copy the example configuration file
 24 | 
 25 | ```bash
 26 | # From the repo's root path
 27 | cp group_vars/example-all.yaml group_vars/all.yaml
 28 | ```
 29 | 
 30 | Fill in the `group_vars/all.yaml` with values that are appropriate for your environment.
 31 | 
 32 | ## Create Kubernetes node VM(s)
 33 | 
 34 | By default, `create-k8s-vm.yaml` will tag the VM with **kube_node**, **kube_control_plane**, and **etcd**
 35 | 
 36 | * **kube_node** : Kubernetes nodes where pods will run.
 37 | * **kube_control_plane** : Where Kubernetes control plane components (apiserver, scheduler, controller) will run.
 38 | * **etcd**: The etcd server. You should have 3 servers for failover purpose.
 39 | 
 40 | For high-availability, it is recommended to have 3+ nodes with these roles.
 41 | See [Kubespray's documentation](https://github.com/kubernetes-sigs/kubespray/blob/v2.27.0/docs/ansible/inventory.md) for more details.
 42 | 
 43 | To adjust the role(s) of the VM, adjust the tags:
 44 | 
 45 | ```bash
 46 | # worker, control plane, and etcd node
 47 | ansible-playbook create-k8s-vm.yaml -K
 48 | 
 49 | # control plane and etcd node
 50 | ansible-playbook create-k8s-vm.yaml --extra-vars "vm_tags=kube_control_plane,etcd" -K
 51 | 
 52 | # worker node
 53 | ansible-playbook create-k8s-vm.yaml --extra-vars "vm_tags=kube_node" -K
 54 | 
 55 | # control plane node
 56 | ansible-playbook create-k8s-vm.yaml --extra-vars "vm_tags=kube_control_plane" -K
 57 | 
 58 | # etcd node
 59 | ansible-playbook create-k8s-vm.yaml --extra-vars "vm_tags=etcd" -K
 60 | ```
 61 | 
 62 | # Deploy Kubernetes via Kubespray
 63 | 
 64 | ## Copy sample inventory files
 65 | 
 66 | ```bash
 67 | DOCKER_IMAGE=quay.io/kubespray/kubespray:v2.27.0
 68 | 
 69 | CID=$(docker create "${DOCKER_IMAGE}")
 70 | docker cp "${CID}:/kubespray/inventory/sample" mycluster
 71 | docker rm "${CID}"
 72 | ```
 73 | 
 74 | Make any desired modifications to the files in "mycluster"
 75 | 
 76 | ## Run deploy
 77 | 
 78 | ```bash
 79 | docker run --rm --name kubespray --tty --interactive \
 80 |     -v "$(pwd)/mycluster:/kubespray/inventory/mycluster" \
 81 |     -v "$(pwd)/group_vars/all.yaml:/kubespray/inventory/group_vars/all.yaml" \
 82 |     -v "$(pwd)/inventories/proxmox.py:/kubespray/inventory/mycluster/proxmox.py" \
 83 |     -v "${HOME}/.ssh:/root/.ssh" \
 84 |     "${DOCKER_IMAGE}" bash
 85 | 
 86 | # Inside the container, run:
 87 | chown $(whoami) ~/.ssh/config
 88 | ansible-playbook -i inventory/mycluster/proxmox.py --user=user --become --become-user=root cluster.yml
 89 | exit
 90 | 
 91 | # Set your SSH config permissions back
 92 | sudo chown $(whoami) ~/.ssh/config
 93 | sudo chown $(whoami) ~/.ssh/known_hosts
 94 | ```
 95 | 
 96 | # kubectl
 97 | 
 98 | Install [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
 99 | 
100 | ```
101 | The Kubernetes command-line tool, kubectl, allows you to run commands against Kubernetes clusters. You can use kubectl to deploy applications, inspect and manage cluster resources, and view logs.
102 | ```
103 | 
104 | ## Authorization
105 | 
106 | ```bash
107 | IP=192.0.1.2  # Controller node IP address
108 | ssh user@${IP} 'mkdir -p "${HOME}/.kube" && sudo cp -f /etc/kubernetes/admin.conf "${HOME}/.kube/config"'
109 | 
110 | mkdir -p "${HOME}/.kube"
111 | ssh user@${IP} 'sudo cat "${HOME}/.kube/config"' > "${HOME}/.kube/config"
112 | sudo chown $(id -u):$(id -g) "${HOME}/.kube/config"
113 | sed -i "s/127.0.0.1/${IP}/g" "${HOME}/.kube/config"
114 | chmod 400 ~/.kube/config
115 | 
116 | kubectl get nodes
117 | ```
118 | 


--------------------------------------------------------------------------------
/ansible.cfg:
--------------------------------------------------------------------------------
1 | [defaults]
2 | host_key_checking = False
3 | inventory = inventories/proxmox.py
4 | 


--------------------------------------------------------------------------------
/create-k8s-vm.yaml:
--------------------------------------------------------------------------------
  1 | - hosts: proxmox
  2 |   become_method: sudo
  3 |   become_user: root
  4 |   become: true
  5 | 
  6 |   tasks:
  7 |   - name: Get local SSH public key
  8 |     ansible.builtin.shell: |
  9 |       ssh-agent sh -c 'ssh-add > /dev/null 2>&1 ; ssh-add -L'
 10 |     register: ssh_public_key
 11 |     become: false
 12 |     delegate_to: localhost
 13 |   
 14 |   - name: Set SSH public key
 15 |     ansible.builtin.set_fact:
 16 |       ssh_public_key: "{{ ssh_public_key.stdout }}"
 17 | 
 18 |   - name: Determine ISO download path
 19 |     ansible.builtin.shell: |
 20 |       pvesh get /storage/{{ proxmox_iso_storage }} --output-format json
 21 |     register: iso_storage
 22 | 
 23 |   - name: Set ISO download path
 24 |     ansible.builtin.set_fact:
 25 |       iso_path: "{{ (iso_storage.stdout | from_json).path }}/template/iso"
 26 | 
 27 |   - name: Download Ubuntu cloud image
 28 |     ansible.builtin.get_url:
 29 |       url: https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64.img
 30 |       dest: "{{ iso_path }}/ubuntu-24.04-minimal-cloudimg-amd64.img"
 31 |       checksum: sha256:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS
 32 | 
 33 |   - name: Generate VM name
 34 |     ansible.builtin.shell: |
 35 |       echo "k8s-node-$(tr -dc a-z0-9 </dev/urandom | head -c 5)"
 36 |     register: vm_name
 37 |   
 38 |   - name: Set VM name
 39 |     ansible.builtin.set_fact:
 40 |       vm_name: "{{ vm_name.stdout }}"
 41 | 
 42 |   - name: Get next VMID
 43 |     ansible.builtin.shell: |
 44 |       pvesh get /cluster/nextid
 45 |     register: next_vmid
 46 | 
 47 |   - name: Set VMID
 48 |     ansible.builtin.set_fact:
 49 |       vmid: "{{ next_vmid.stdout }}"
 50 | 
 51 |   - name: Create new VM
 52 |     ansible.builtin.shell: |
 53 |       qm create {{ vmid }} \
 54 |         --agent 1 \
 55 |         --cores {{ vm_cores }} \
 56 |         --ipconfig0 ip=dhcp \
 57 |         --memory {{ vm_memory }} \
 58 |         --name {{ vm_name }} \
 59 |         --net0 virtio,bridge=vmbr0 \
 60 |         --serial0 socket \
 61 |         --tags "{{ vm_tags }}" \
 62 |         --vga serial0
 63 | 
 64 |   - name: Convert cloud image to disk
 65 |     ansible.builtin.shell: |
 66 |       qm importdisk {{ vmid }} "{{ iso_path }}/ubuntu-24.04-minimal-cloudimg-amd64.img" {{ proxmox_disk_storage }}
 67 |     register: disk_location
 68 | 
 69 |   - name: Set disk location
 70 |     ansible.builtin.shell: |
 71 |       echo "{{ disk_location.stdout }}" | tail -n 1 | grep -o -E "'.*'" | sed "s/'//g" | sed 's/unused[0-9]*://g'
 72 |     register: disk_location
 73 | 
 74 |   - name: Assign to to SCSi drive on VM
 75 |     ansible.builtin.shell: |
 76 |       qm set {{ vmid }} --scsihw virtio-scsi-pci --scsi0 {{ disk_location.stdout }}
 77 | 
 78 |   - name: Resize disk
 79 |     ansible.builtin.shell: |
 80 |       qm resize {{ vmid }} scsi0 {{ vm_disk_size }}
 81 | 
 82 |   - name: Set boot priority
 83 |     ansible.builtin.shell: |
 84 |       qm set {{ vmid }} --boot c --bootdisk scsi0
 85 | 
 86 |   - name: Enable snippets
 87 |     ansible.builtin.shell: |
 88 |       pvesm set {{ proxmox_snippet_storage }} --content images,rootdir,vztmpl,backup,iso,snippets
 89 | 
 90 |   - name: Create cloud-init drive
 91 |     ansible.builtin.shell: |
 92 |       qm set {{ vmid }} --scsi1 {{ proxmox_snippet_storage }}:cloudinit
 93 | 
 94 |   - name: Determine snippets path
 95 |     ansible.builtin.shell: |
 96 |       pvesh get /storage/{{ proxmox_snippet_storage }} --output-format json
 97 |     register: snippets_storage
 98 | 
 99 |   - name: Set ISO download path
100 |     ansible.builtin.set_fact:
101 |       snippets_path: "{{ (snippets_storage.stdout | from_json).path }}/snippets"
102 | 
103 |   - name: Set cloud-init config file
104 |     ansible.builtin.copy:
105 |       dest: "{{ snippets_path }}/{{ vm_name }}-config.yaml"
106 |       content: |
107 |         #cloud-config
108 |         hostname: {{ vm_name }}
109 |         manage_etc_hosts: true
110 |         user: {{ vm_user }}
111 |         password: {{ vm_password }}
112 |         chpasswd:
113 |           expire: False
114 |         ssh_authorized_keys:
115 |           - {{ ssh_public_key }}
116 |         users:
117 |           - default
118 |         package_upgrade: true
119 |         packages:
120 |           - qemu-guest-agent
121 |         runcmd:
122 |           - sudo sed -i 's/^[# ]*PasswordAuthentication .*/PasswordAuthentication no/g' /etc/ssh/sshd_config
123 |           - sudo sed -i 's/^[# ]*PermitRootLogin .*/PermitRootLogin no/g' /etc/ssh/sshd_config
124 |           - sudo rm /etc/machine-id
125 |           - sudo rm /var/lib/dbus/machine-id
126 |           - sudo truncate -s 0 /etc/machine-id
127 |           - sudo ln -s /etc/machine-id /var/lib/dbus/machine-id
128 |           - shutdown
129 | 
130 |   - name: Mount cloud-init file
131 |     ansible.builtin.shell: |
132 |       qm set {{ vmid }} --cicustom "user={{ proxmox_snippet_storage }}:snippets/{{ vm_name }}-config.yaml"
133 | 
134 |   - name: Start VM
135 |     ansible.builtin.shell: |
136 |       qm start {{ vmid }}
137 | 
138 |   - name: Wait for VM setup to complete
139 |     ansible.builtin.shell: |
140 |       qm status {{ vmid }}
141 |     register: vm_status
142 |     retries: 50
143 |     delay: 10
144 |     until: '"stopped" in vm_status.stdout'
145 | 
146 |   - name: Unmount cloud-init drive
147 |     ansible.builtin.shell: |
148 |       qm disk unlink {{ vmid }} --idlist scsi1
149 | 
150 |   - name: Clean up cloud-init file
151 |     ansible.builtin.file:
152 |       path: "{{ snippets_path }}/{{ vm_name }}-config.yaml"
153 |       state: absent
154 | 
155 |   - name: Snapshot VM
156 |     ansible.builtin.shell: |
157 |       qm snapshot {{ vmid }} base
158 | 
159 |   - name: Start VM
160 |     ansible.builtin.shell: |
161 |       qm start {{ vmid }}
162 | 


--------------------------------------------------------------------------------
/group_vars/example-all.yaml:
--------------------------------------------------------------------------------
 1 | proxmox_ip_address: 192.0.1.1
 2 | proxmox_username: root@pam
 3 | proxmox_password: password
 4 | proxmox_ssh_port: 22
 5 | proxmox_https_port: 8006
 6 | proxmox_trust_invalid_https_certs: true
 7 | 
 8 | # Used to provision VMs
 9 | vm_disk_size: 64G
10 | vm_memory: 4096
11 | vm_cores: 2
12 | vm_user: user
13 | vm_password: password
14 | vm_tags: "kube_control_plane, etcd, kube_node"
15 | proxmox_disk_storage: local-lvm
16 | proxmox_iso_storage: local
17 | proxmox_snippet_storage: local
18 | 


--------------------------------------------------------------------------------
/inventories/proxmox.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | 
  3 | from pathlib import Path
  4 | from typing import Dict, List
  5 | import http.client
  6 | import json
  7 | import re
  8 | import ssl
  9 | 
 10 | 
 11 | class PromoxHost:
 12 |     def __init__(self):
 13 |         ansible_vars_file = Path(Path(__file__).parent.parent, "group_vars", "all.yaml")
 14 | 
 15 |         with open(ansible_vars_file) as f:
 16 |             ansible_vars_content = f.read()
 17 | 
 18 |         self.ip = re.search("proxmox_ip_address: (.*)", ansible_vars_content)[1]
 19 |         self.username = re.search("proxmox_username: (.*)", ansible_vars_content)[1]
 20 |         self.password = re.search("proxmox_password: (.*)", ansible_vars_content)[1]
 21 |         self.ssh_port = re.search("proxmox_ssh_port: (.*)", ansible_vars_content)[1]
 22 |         self.https_port = re.search("proxmox_https_port: (.*)", ansible_vars_content)[1]
 23 |         self.trust_invalid_https_certs = re.search(
 24 |             "proxmox_trust_invalid_https_certs: (.*)", ansible_vars_content
 25 |         )[1]
 26 | 
 27 | 
 28 | class ProxmoxVM:
 29 |     def __init__(self, node: str, meta: Dict, network_interfaces: Dict):
 30 |         self.node = node
 31 |         self.meta = meta
 32 |         self.network_interfaces = network_interfaces
 33 | 
 34 |     @property
 35 |     def name(self):
 36 |         return self.meta["name"]
 37 | 
 38 |     @property
 39 |     def tags(self):
 40 |         tags = [tag.strip() for tag in self.meta["tags"].split(",")]
 41 |         return tags
 42 | 
 43 |     @property
 44 |     def ip(self):
 45 |         if not self.network_interfaces:
 46 |             return None
 47 | 
 48 |         for interface in self.network_interfaces:
 49 |             if interface["name"] == "lo":
 50 |                 continue
 51 | 
 52 |             for ip in interface["ip-addresses"]:
 53 |                 if ip["ip-address-type"] == "ipv6":
 54 |                     continue
 55 | 
 56 |                 # Return first non-loopback ipv4 address
 57 |                 return ip["ip-address"]
 58 | 
 59 |     @property
 60 |     def running(self):
 61 |         return self.meta["status"] == "running"
 62 | 
 63 | 
 64 | class ProxmoxAPI:
 65 |     def __init__(self, host: PromoxHost):
 66 |         self.host = host
 67 |         self.url = f"{host.ip}:{host.https_port}"
 68 |         self.auth_cookie = self.auth()
 69 | 
 70 |     def auth(self):
 71 |         conn = http.client.HTTPSConnection(
 72 |             self.url, context=ssl._create_unverified_context(), timeout=10
 73 |         )
 74 |         headers = {
 75 |             "Content-Type": "application/x-www-form-urlencoded",
 76 |         }
 77 | 
 78 |         conn.request(
 79 |             "POST",
 80 |             "/api2/json/access/ticket",
 81 |             f"username={self.host.username}&password={self.host.password}",
 82 |             headers,
 83 |         )
 84 | 
 85 |         res = conn.getresponse()
 86 |         data = res.read()
 87 |         return json.loads(data.decode("utf-8"))["data"]
 88 | 
 89 |     def get(self, url: str, data=None):
 90 |         conn = http.client.HTTPSConnection(
 91 |             self.url, context=ssl._create_unverified_context(), timeout=10
 92 |         )
 93 |         headers = {"Cookie": f"PVEAuthCookie={self.auth_cookie['ticket']}"}
 94 |         conn.request("GET", url, body=data, headers=headers)
 95 |         res = conn.getresponse()
 96 |         data = res.read()
 97 | 
 98 |         return json.loads(data.decode("utf-8"))["data"]
 99 | 
100 |     def get_vms(self) -> List[ProxmoxVM]:
101 |         vms = []
102 | 
103 |         for node in self.get("/api2/json/nodes"):
104 |             node_vms = self.get(f"/api2/json/nodes/{node['node']}/qemu")
105 | 
106 |             for node_vm in node_vms:
107 |                 response = self.get(
108 |                     f"/api2/json/nodes/{node['node']}/qemu/{node_vm['vmid']}/agent/network-get-interfaces"
109 |                 )
110 |                 # Return 'None' in case VM isn't running
111 |                 network_interfaces = response["result"] if response else None
112 | 
113 |                 vm = ProxmoxVM(node["node"], node_vm, network_interfaces)
114 |                 vms.append(vm)
115 | 
116 |         return vms
117 | 
118 | 
119 | if __name__ == "__main__":
120 |     proxmox = PromoxHost()
121 | 
122 |     inventory = {
123 |         "_meta": {
124 |             "hostvars": {
125 |                 "proxmox": {
126 |                     "ansible_host": proxmox.ip,
127 |                     "ansible_port": proxmox.ssh_port,
128 |                 }
129 |             }
130 |         },
131 |         "all": {"hosts": ["proxmox"], "children": []},
132 |     }
133 | 
134 |     for vm in ProxmoxAPI(proxmox).get_vms():
135 |         if not vm.running or not vm.ip:
136 |             continue
137 | 
138 |         inventory["_meta"]["hostvars"][vm.name] = {"ansible_host": vm.ip}
139 | 
140 |         for tag in vm.tags:
141 |             if tag not in inventory.keys():
142 |                 inventory["all"]["children"].append(tag)
143 |                 inventory[tag] = {"hosts": [vm.name]}
144 |             else:
145 |                 inventory[tag]["hosts"].append(vm.name)
146 | 
147 |     print((json.dumps(inventory, indent=True)))
148 | 


--------------------------------------------------------------------------------