├── proxmox ├── vm_definitions │ └── dev.yaml ├── locals.tf ├── main.tf ├── virtual_machines.tf └── variables.tf ├── envexample ├── gitlab-ci.yml ├── Makefile └── README.md /proxmox/vm_definitions/dev.yaml: -------------------------------------------------------------------------------- 1 | dev: 2 | vmid: 2000 3 | cores: 12 4 | memory: 64000 5 | ipconfig0: "ip={machine_ip_address_here}/24,gw={gateway_ip}" 6 | disk: 7 | size: "101580M" -------------------------------------------------------------------------------- /proxmox/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_configs = { 3 | for x in fileset("./vm_definitions/", "*") : trimsuffix(x, ".yaml") => merge(var.DEFAULT_VM_SETTINGS, yamldecode(file("./vm_definitions/${x}"))[trimsuffix(x, ".yaml")]) 4 | } 5 | configs = { 6 | for id, cfg in local.base_configs : id => merge(cfg, { "disk" : merge(var.DEFAULT_DISK, lookup(cfg, "disk", {})) }) 7 | } 8 | } 9 | 10 | -------------------------------------------------------------------------------- /envexample: -------------------------------------------------------------------------------- 1 | export PROXMOX_TF_USER="user" --proxmox service account username 2 | export PROXMOX_TF_PASS="" --proxmox service account password 3 | export PROXMOX_TF_IP=https://{proxmox_IP}/api2/json --can be any one of the cluster ips 4 | export PROXMOX_SSH_KEY='`cat ~/.ssh/id_rsa.pub`' --ssh key for cloud init 5 | export PROXMOX_TOKEN_SECRET="" --proxmox service account token 6 | export PROXMOX_TOKEN_ID="user@pve!password" -- proxmox service account user 7 | 8 | export LOCAL_SSH_FILE="/home/username/.ssh/id_rsa.pub" --path to your ssh 9 | export PRIVATE_SSH_KEY='`cat ~/.ssh/id_rsa`' --path to your ssh key 10 | export CI_PASS="" --password for the cloud init options 11 | 12 | export GIT_PASSWORD="" --gitlab access token -------------------------------------------------------------------------------- /proxmox/main.tf: -------------------------------------------------------------------------------- 1 | 2 | terraform { 3 | backend "http" { 4 | address = "http://{gitlab_url}/api/v4/projects/4/terraform/state/cluster_state" 5 | lock_address = "http://{gitlab_url}/api/v4/projects/4/terraform/state/cluster_state/lock" 6 | unlock_address = "http://{gitlab_url}/api/v4/projects/4/terraform/state/cluster_state/lock" 7 | lock_method = "POST" 8 | unlock_method = "DELETE" 9 | retry_wait_min = 5 10 | } 11 | 12 | required_providers { 13 | proxmox = { 14 | source = "Telmate/proxmox" 15 | version = "2.9.11" 16 | } 17 | } 18 | required_version = ">= 0.13" 19 | } 20 | 21 | provider "proxmox" { 22 | pm_api_url = var.PROXMOX_TF_IP 23 | pm_user = "${var.PROXMOX_TF_USER}@pve" 24 | pm_password = var.PROXMOX_TF_PASS 25 | pm_tls_insecure = "true" 26 | pm_timeout = 3600 27 | } 28 | 29 | -------------------------------------------------------------------------------- /gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - template: Terraform/Base.gitlab-ci.yml 3 | 4 | variables: 5 | TF_STATE_NAME: clusterstate 6 | TF_CACHE_KEY: cluster 7 | 8 | stages: 9 | - init 10 | - validate 11 | - plan 12 | - deploy 13 | 14 | init: 15 | before_script: 16 | - cd ~/ 17 | - rm -r -f {gitlab_project_folder} 18 | - git clone http://oauth2:${INFRA_ACCESS_TOKEN}@{gitlab_project_url}.git 19 | - cd {gitlab_project_folder} 20 | extends: .init 21 | script: 22 | - make proxmox-tf-init 23 | 24 | validate: 25 | before_script: 26 | - cd ~/{gitlab_project_folder} 27 | extends: .validate 28 | needs: 29 | - job: init 30 | script: 31 | - make proxmox-tf-validate 32 | 33 | 34 | plan: 35 | before_script: 36 | - cd ~/{gitlab_project_folder} 37 | extends: .plan 38 | needs: [validate, init] 39 | script: 40 | - make proxmox-tf-plan 41 | 42 | deploy: 43 | before_script: 44 | - cd ~/{gitlab_project_folder} 45 | extends: .deploy 46 | needs: [plan] 47 | script: 48 | - make proxmox-tf-apply-gitlab 49 | when: manual 50 | after_script: 51 | - cd ~/ 52 | - rm -r {gitlab_project_folder} 53 | 54 | -------------------------------------------------------------------------------- /proxmox/virtual_machines.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "proxmox_vm_qemu" "proxmox_vm" { 3 | for_each = { for name, settings in local.configs : name => settings } 4 | name = each.key 5 | vmid = each.value.vmid 6 | agent = 1 7 | define_connection_info = true 8 | cloudinit_cdrom_storage = "local-lvm" 9 | target_node = each.value.target_node 10 | clone = each.value.clone 11 | full_clone = each.value.full_clone 12 | os_type = "cloud-init" 13 | cores = each.value.cores 14 | sockets = each.value.sockets 15 | cpu = each.value.cpu 16 | memory = each.value.memory 17 | scsihw = each.value.scsihw 18 | bootdisk = each.value.bootdisk 19 | desc = each.value.desc 20 | ssh_user = "nohbdy" 21 | vga { 22 | type = "serial0" 23 | } 24 | 25 | disk { 26 | slot = each.value.disk.slot 27 | size = each.value.disk.size 28 | type = each.value.disk.type 29 | storage = each.value.disk.storage 30 | iothread = each.value.disk.iothread 31 | } 32 | 33 | network { 34 | model = "virtio" 35 | bridge = "vmbr0" 36 | } 37 | 38 | lifecycle { 39 | ignore_changes = [] 40 | } 41 | os_network_config = each.value.os_network_config 42 | # Cloud Init Settings 43 | ipconfig0 = each.value.ipconfig0 44 | sshkeys = < CI/CD > Variables. 33 | 34 | Also, take a look through the `variables.tf` and update the URLs to match your gitlab/project API urls. 35 | 36 | ### Makefile usage 37 | 38 | Since there are a lot of `TF_VAR`s I chose to setup a makefile with the commands I've used to mess with terraform. One 39 | thing I had to do to migrate to this type of terraform repo structure was `mv` all the tf resources from `proxmox_vm_qemu.{vmname}` 40 | to `proxmox_vm_qemu.proxmox_vm[\"vmname\"]`, which you can do with the makefile command: 41 | 42 | ```shell 43 | make proxmox-tf-state-move -e from="'proxmox_vm_qemu.{vmname}'" to="'proxmox_vm_qemu.proxmox_vm[\"vmname\"]'" 44 | ``` 45 | 46 | So keep this in mind if you are moving to this type of setup. 47 | 48 | Also, if you currently have VMs defined and want to move to using terraform to manage them, you can use the `proxmox-tf-import` make target. 49 | For these I had an issue where terraform wanted to destroy and recreate the VMs, so I had to add the attributes that terraform 50 | said required recreate to the `lifecycle` block on line 38 of the `virtual_machines.tf` so read through the terraform plan 51 | carefully. 52 | 53 | ### Defining New VMs 54 | 55 | Virtual machine definitions are added as .yaml files under proxmox/vm_definitions. To see the options 56 | available for the yaml configuration, look at the `virtual_machines.tf` resource definition (everything that 57 | has `each.value`). 58 | 59 | The yaml definition gets combined with the default configuration for the VM and disk starting at line 71 60 | of the `variables.tf`. You can update the defaults to match whatever makes sense for your proxmox setup. For 61 | example, I created a VM with ubuntu and setup cloud init on that, and turned it into a template called 'ubuntu-cloudinit' 62 | defined on line 92 of `variables.tf`. 63 | 64 | The only required attributes you need for the yaml definition in order for this to work properly are: 65 | 66 | - vmid 67 | - ipconfig0 68 | 69 | but you can add any level of detail per configuration to fit your needs. 70 | 71 | Example: 72 | 73 | ```yaml 74 | dev: 75 | vmid: 2000 76 | cores: 12 77 | memory: 64000 78 | ipconfig0: "ip={machine_ip_address_here}/24,gw={gateway_ip}" 79 | disk: 80 | size: "101580M" 81 | 82 | ``` 83 | 84 | > Note: the .yaml file name should be the first line of the yaml file, so the first line of `dev.yaml` is `dev:` 85 | 86 | ## Deploying Proxmox VMs 87 | 88 | Locally, in the main repo you can run the following commands to deploy Nomad VMs 89 | to Proxmox: 90 | 91 | First, initialize terraform: 92 | ```shell 93 | make proxmox_modules-tf-init 94 | ``` 95 | 96 | After perform the terraform plan 97 | 98 | ```shell 99 | make proxmox_modules-tf-plan 100 | ``` 101 | 102 | Finally, deploy the plan 103 | ```shell 104 | make proxmox_modules-tf-apply 105 | ``` 106 | 107 | You can also tear everything down to start from scratch if needed 108 | ```shell 109 | make proxmox_modules-tf-destroy 110 | ``` 111 | 112 | If you add/edit any of the terraform configurations, remember to reformat before committing/pushing 113 | 114 | ```shell 115 | terraform fmt 116 | ``` 117 | 118 | ## Setting up gitlab ci/cd 119 | 120 | If you want to setup the gitlab ci/cd pipeline, you'll need to add these environemnt variables under 121 | Settings > CI/CD > Variables. 122 | 123 | This project uses gitlab-managed terraform state, so you will need to get that setup first. 124 | 125 | Check the `.gitlab-ci.yml` and update `{gitlab_project_folder}` and `{gitlab_project_url}`. I was having an issue with 126 | the caching on my gitlab runner, so my solution was to have the runner clone the project and execute the `make` command 127 | from that folder each time. 128 | 129 | --------------------------------------------------------------------------------