├── .github └── workflows │ └── main.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── cmd ├── common │ └── common.go ├── kubernetes │ ├── api │ │ ├── api.go │ │ └── v1alpha1 │ │ │ └── qemu.go │ ├── client.go │ └── qemu.go └── proxmox │ ├── client.go │ ├── cluster.go │ ├── node.go │ ├── qemu.go │ └── storage.go ├── config.yaml ├── examples └── example-qemu.yaml ├── go.mod ├── go.sum ├── leaderElection.go ├── main.go └── utils.go /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | env: 8 | REGISTRY: "crashntech/${{ github.event.repository.name }}" 9 | 10 | jobs: 11 | build_and_push: 12 | name: Push Docker image to Docker Hub 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Check out the repo 16 | uses: actions/checkout@v4 17 | 18 | - name: Log in to Docker Hub 19 | uses: docker/login-action@v3 20 | with: 21 | username: ${{ secrets.REGISTRY_USERNAME }} 22 | password: ${{ secrets.REGISTRY_TOKEN }} 23 | 24 | - name: Extract metadata (tags, labels) for Docker 25 | id: meta 26 | uses: docker/metadata-action@v5 27 | with: 28 | images: ${{ env.REGISTRY }} 29 | 30 | - name: Build and push Docker image 31 | uses: docker/build-push-action@v5 32 | with: 33 | context: . 34 | file: ./Dockerfile 35 | push: true 36 | tags: ${{ steps.meta.outputs.tags }} 37 | labels: ${{ steps.meta.outputs.labels }} 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine3.17 as builder 2 | 3 | COPY cmd/ /app/cmd 4 | COPY main.go /app/main.go 5 | COPY utils.go /app/utils.go 6 | COPY leaderElection.go /app/leaderElection.go 7 | COPY go.mod /app/go.mod 8 | COPY go.sum /app/go.sum 9 | WORKDIR /app 10 | RUN go build -o proxmox-operator 11 | 12 | FROM alpine:3.17 13 | COPY --from=builder /app/proxmox-operator /app/proxmox-operator 14 | WORKDIR /app 15 | CMD /app/proxmox-operator 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # proxmox-operator 2 | 3 | This repository provides a Kubernetes operator for managing [Proxmox virtualization platform](https://pve.proxmox.com/pve-docs/) QEMU VMs via CRDs. 4 | 5 | ## Getting Started 6 | 7 | * `git clone https://github.com/CRASH-Tech/proxmox-operator.git` 8 | * `cd proxmox-operator` 9 | * Edit charts/proxmox-operator/values.yaml for your environment 10 | * `helm repo add crash-tech https://crash-tech.github.io/charts/` 11 | * `helm install proxmox-operator crash-tech/proxmox-operator -f charts/proxmox-operator/values.yaml` 12 | 13 | ## Deploy example VM 14 | 15 | * Edit examples/example-qemu.yaml for your environment 16 | * `kubectl apply -f examples/example-qemu.yaml` 17 | * Check VM status 18 | * `kubectl get qemu` 19 | 20 | ## Example VM 21 | ``` 22 | apiVersion: proxmox.xfix.org/v1alpha1 23 | kind: Qemu 24 | metadata: 25 | name: example-qemu 26 | finalizers: 27 | - resources-finalizer.proxmox-operator.xfix.org 28 | spec: 29 | cluster: pve-test 30 | #node: crash-lab ### If not set it will set automaticly from "pool" 31 | #vmid: 222 ### If not set it will set automaticly 32 | pool: prod ### Cluster pool for place VM 33 | anti-affinity: "" ### The anti-affinity group. VM's with same anti-affinity group will be placed on different nodes 34 | autostart: true 35 | autostop: true 36 | cpu: 37 | type: host 38 | sockets: 2 39 | cores: 1 40 | memory: 41 | size: 2048 42 | balloon: 2048 43 | network: 44 | net0: 45 | model: virtio 46 | #mac: A2:7B:45:48:9C:E6 ### If not set it will set automaticly 47 | bridge: vmbr0 48 | tag: 103 49 | disk: 50 | scsi0: 51 | storage: local-lvm 52 | size: 9G 53 | tags: 54 | - test1 55 | - test2 56 | options: 57 | ostype: "l26" 58 | bios: "seabios" 59 | smbios1: "uuid=3ae878b3-a77e-4a4a-adc6-14ee88350d36,manufacturer=MTIz,product=MTIz,version=MTIz,serial=MTIz,sku=MTIz,family=MTIz,base64=1" 60 | scsihw: "virtio-scsi-pci" 61 | boot: "order=net0;ide2;scsi0" 62 | ide2: "none,media=cdrom" 63 | hotplug: "network,disk,usb" 64 | tablet: 1 65 | onboot: 0 66 | kvm: 1 67 | agent: "0" 68 | numa: 1 69 | protection: 0 70 | ``` 71 | 72 | ## Useful links 73 | 74 | * [Kubernetes CRD](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) 75 | * [Proxmox](https://www.proxmox.com/en/) 76 | * [Proxmox documentation](https://pve.proxmox.com/pve-docs/) 77 | -------------------------------------------------------------------------------- /cmd/common/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import ( 20 | "github.com/CRASH-Tech/proxmox-operator/cmd/proxmox" 21 | "k8s.io/client-go/dynamic" 22 | "k8s.io/client-go/kubernetes" 23 | ) 24 | 25 | type Config struct { 26 | Log LogConfig `yaml:"log"` 27 | Clusters map[string]proxmox.ClusterApiConfig `yaml:"clusters"` 28 | DynamicClient *dynamic.DynamicClient 29 | KubernetesClient *kubernetes.Clientset 30 | Listen string `yaml:"listen"` 31 | } 32 | 33 | type LogConfig struct { 34 | Level string `yaml:"level"` 35 | Format string `yaml:"format"` 36 | } 37 | -------------------------------------------------------------------------------- /cmd/kubernetes/api/api.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package api 18 | 19 | type CustomResource struct { 20 | APIVersion string `json:"apiVersion"` 21 | Kind string `json:"kind"` 22 | Metadata CustomResourceMetadata `json:"metadata"` 23 | } 24 | 25 | type CustomResourceMetadata struct { 26 | Name string `json:"name"` 27 | Uid string `json:"uid"` 28 | Generation int `json:"generation"` 29 | ResourceVersion string `json:"resourceVersion"` 30 | CreationTimestamp string `json:"creationTimestamp"` 31 | DeletionGracePeriodSeconds int `json:"deletionGracePeriodSeconds,omitempty"` 32 | DeletionTimestamp string `json:"deletionTimestamp,omitempty"` 33 | Finalizers []string `json:"finalizers"` 34 | } 35 | 36 | func (cr *CustomResource) RemoveFinalizers() { 37 | cr.Metadata.Finalizers = []string{} 38 | } 39 | -------------------------------------------------------------------------------- /cmd/kubernetes/api/v1alpha1/qemu.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import "github.com/CRASH-Tech/proxmox-operator/cmd/kubernetes/api" 20 | 21 | const ( 22 | STATUS_QEMU_EMPTY = "" 23 | STATUS_QEMU_SYNCED = "SYNCED" 24 | STATUS_QEMU_OUT_OF_SYNC = "OUT OF SYNC" 25 | STATUS_QEMU_PENDING = "PENDING" 26 | STATUS_QEMU_CLONING = "CLONING" 27 | STATUS_QEMU_DELETING = "DELETING" 28 | STATUS_QEMU_UNKNOWN = "UNKNOWN" 29 | 30 | STATUS_POWER_ON = "ON" 31 | STATUS_POWER_OFF = "OFF" 32 | STATUS_POWER_UNKNOWN = "UNKNOWN" 33 | ) 34 | 35 | type Qemu struct { 36 | *api.CustomResource 37 | Spec QemuSpec `json:"spec"` 38 | Status QemuStatus `json:"status"` 39 | } 40 | 41 | type QemuSpec struct { 42 | Autostart bool `json:"autostart"` 43 | Autostop bool `json:"autostop"` 44 | Cluster string `json:"cluster"` 45 | Node string `json:"node"` 46 | Pool string `json:"pool"` 47 | Clone string `json:"clone"` 48 | AntiAffinity string `json:"anti-affinity"` 49 | VmId int `json:"vmid"` 50 | CPU QemuCPU `json:"cpu"` 51 | Memory QemuMemory `json:"memory"` 52 | Disk map[string]QemuDisk `json:"disk"` 53 | Network map[string]QemuNetwork `json:"network"` 54 | Options map[string]interface{} `json:"options"` 55 | Tags []string `json:"tags"` 56 | } 57 | 58 | type QemuCPU struct { 59 | Cores int `json:"cores"` 60 | Sockets int `json:"sockets"` 61 | Type string `json:"type"` 62 | } 63 | 64 | type QemuDisk struct { 65 | Size string `json:"size"` 66 | Storage string `json:"storage"` 67 | } 68 | 69 | type QemuMemory struct { 70 | Balloon int64 `json:"balloon"` 71 | Size int64 `json:"size"` 72 | } 73 | 74 | type QemuNetwork struct { 75 | Bridge string `json:"bridge"` 76 | Mac string `json:"mac"` 77 | Model string `json:"model"` 78 | Tag int `json:"tag"` 79 | } 80 | 81 | type QemuStatus struct { 82 | Status string `json:"status"` 83 | Power string `json:"power"` 84 | Cluster string `json:"cluster"` 85 | Node string `json:"node"` 86 | VmId int `json:"vmid"` 87 | Net []QemuStatusNetwork `json:"net"` 88 | } 89 | 90 | type QemuStatusNetwork struct { 91 | Name string `json:"name"` 92 | Mac string `json:"mac"` 93 | } 94 | -------------------------------------------------------------------------------- /cmd/kubernetes/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubernetes 18 | 19 | import ( 20 | "context" 21 | 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 24 | "k8s.io/apimachinery/pkg/runtime/schema" 25 | "k8s.io/apimachinery/pkg/types" 26 | "k8s.io/client-go/dynamic" 27 | ) 28 | 29 | type Client struct { 30 | ctx context.Context 31 | dynamic dynamic.DynamicClient 32 | } 33 | 34 | type V1alpha1 struct { 35 | client *Client 36 | } 37 | 38 | func NewClient(ctx context.Context, dynamic dynamic.DynamicClient) *Client { 39 | client := Client{ 40 | ctx: ctx, 41 | dynamic: dynamic, 42 | } 43 | 44 | return &client 45 | } 46 | 47 | func (client *Client) dynamicGet(resourceId schema.GroupVersionResource, name string) ([]byte, error) { 48 | 49 | item, err := client.dynamic.Resource(resourceId).Get(client.ctx, name, metav1.GetOptions{}) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | jsonData, err := item.MarshalJSON() 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | return jsonData, nil 60 | } 61 | 62 | func (client *Client) dynamicGetAll(resourceId schema.GroupVersionResource) ([][]byte, error) { 63 | 64 | items, err := client.dynamic.Resource(resourceId).List(client.ctx, metav1.ListOptions{}) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | var result [][]byte 70 | for _, item := range items.Items { 71 | jsonData, err := item.MarshalJSON() 72 | if err != nil { 73 | return nil, err 74 | } 75 | result = append(result, jsonData) 76 | } 77 | 78 | return result, nil 79 | } 80 | 81 | func (client *Client) dynamicPatch(resourceId schema.GroupVersionResource, name string, patch []byte) ([]byte, error) { 82 | 83 | item, err := client.dynamic.Resource(resourceId).Patch(client.ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | jsonData, err := item.MarshalJSON() 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | return jsonData, nil 94 | } 95 | 96 | func (client *Client) dynamicUpdateStatus(resourceId schema.GroupVersionResource, name string, patch []byte) ([]byte, error) { 97 | var data unstructured.Unstructured 98 | err := data.UnmarshalJSON(patch) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | result, err := client.dynamic.Resource(resourceId).UpdateStatus(client.ctx, &data, metav1.UpdateOptions{}) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | jsonData, err := result.MarshalJSON() 109 | if err != nil { 110 | return nil, err 111 | } 112 | 113 | return jsonData, nil 114 | } 115 | 116 | func (client *Client) V1alpha1() *V1alpha1 { 117 | result := V1alpha1{ 118 | client: client, 119 | } 120 | 121 | return &result 122 | } 123 | 124 | func (v1alpha1 *V1alpha1) Qemu() *Qemu { 125 | qemu := Qemu{ 126 | client: v1alpha1.client, 127 | resourceId: schema.GroupVersionResource{ 128 | Group: "proxmox.xfix.org", 129 | Version: "v1alpha1", 130 | Resource: "qemu", 131 | }, 132 | } 133 | 134 | return &qemu 135 | } 136 | -------------------------------------------------------------------------------- /cmd/kubernetes/qemu.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubernetes 18 | 19 | import ( 20 | "encoding/json" 21 | 22 | "github.com/CRASH-Tech/proxmox-operator/cmd/kubernetes/api/v1alpha1" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | ) 25 | 26 | var () 27 | 28 | type Qemu struct { 29 | client *Client 30 | resourceId schema.GroupVersionResource 31 | } 32 | 33 | func (qemu *Qemu) Get(name string) (v1alpha1.Qemu, error) { 34 | item, err := qemu.client.dynamicGet(qemu.resourceId, name) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | var result v1alpha1.Qemu 40 | err = json.Unmarshal(item, &result) 41 | if err != nil { 42 | return v1alpha1.Qemu{}, err 43 | } 44 | 45 | return result, nil 46 | } 47 | 48 | func (qemu *Qemu) GetAll() ([]v1alpha1.Qemu, error) { 49 | items, err := qemu.client.dynamicGetAll(qemu.resourceId) 50 | if err != nil { 51 | panic(err) 52 | } 53 | 54 | var result []v1alpha1.Qemu 55 | for _, item := range items { 56 | var q v1alpha1.Qemu 57 | err = json.Unmarshal(item, &q) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | result = append(result, q) 63 | } 64 | 65 | return result, nil 66 | } 67 | 68 | func (qemu *Qemu) Patch(q v1alpha1.Qemu) (v1alpha1.Qemu, error) { 69 | jsonData, err := json.Marshal(q) 70 | if err != nil { 71 | return v1alpha1.Qemu{}, err 72 | } 73 | 74 | resp, err := qemu.client.dynamicPatch(qemu.resourceId, q.Metadata.Name, jsonData) 75 | if err != nil { 76 | return v1alpha1.Qemu{}, err 77 | } 78 | 79 | var result v1alpha1.Qemu 80 | err = json.Unmarshal(resp, &result) 81 | if err != nil { 82 | return v1alpha1.Qemu{}, err 83 | } 84 | 85 | return result, nil 86 | } 87 | 88 | func (qemu *Qemu) UpdateStatus(q v1alpha1.Qemu) (v1alpha1.Qemu, error) { 89 | jsonData, err := json.Marshal(q) 90 | if err != nil { 91 | return v1alpha1.Qemu{}, err 92 | } 93 | 94 | resp, err := qemu.client.dynamicUpdateStatus(qemu.resourceId, q.Metadata.Name, jsonData) 95 | if err != nil { 96 | return v1alpha1.Qemu{}, err 97 | } 98 | 99 | var result v1alpha1.Qemu 100 | err = json.Unmarshal(resp, &result) 101 | if err != nil { 102 | return v1alpha1.Qemu{}, err 103 | } 104 | 105 | return result, nil 106 | } 107 | -------------------------------------------------------------------------------- /cmd/proxmox/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "crypto/tls" 21 | "fmt" 22 | "sort" 23 | 24 | "github.com/go-resty/resty/v2" 25 | log "github.com/sirupsen/logrus" 26 | ) 27 | 28 | const ( 29 | NODE_STATUS_ONLINE = "online" 30 | NODE_STATUS_OFFLINE = "offline" 31 | ) 32 | 33 | type Client struct { 34 | Clusters map[string]ClusterApiConfig 35 | } 36 | 37 | type QemuPlace struct { 38 | Cluster string 39 | Node string 40 | VmId int 41 | Found bool 42 | } 43 | 44 | func NewClient(clusters map[string]ClusterApiConfig) *Client { 45 | client := Client{ 46 | Clusters: clusters, 47 | } 48 | 49 | return &client 50 | } 51 | 52 | func (client *Client) Cluster(cluster string) *Cluster { 53 | apiConfig, isExists := client.Clusters[cluster] 54 | if !isExists { 55 | log.Errorf("unknown cluster: %s", cluster) 56 | } 57 | 58 | restyClient := resty.New() 59 | restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) 60 | restyClient.SetHeader("Content-Type", "application/json") 61 | restyClient.SetHeader("Authorization", fmt.Sprintf("PVEAPIToken=%s=%s", apiConfig.ApiTokenId, apiConfig.ApiTokenSecret)) 62 | 63 | result := Cluster{ 64 | name: cluster, 65 | apiCOnfig: apiConfig, 66 | resty: restyClient, 67 | } 68 | 69 | return &result 70 | } 71 | 72 | func (client *Client) GetQemuPlacableCluster(request PlaceRequest) (QemuPlace, error) { 73 | qemuCount := make(map[string]int) 74 | for cluster, _ := range client.Clusters { 75 | if count, err := client.Cluster(cluster).GetResourceCount(RESOURCE_QEMU); err == nil { 76 | qemuCount[cluster] = count 77 | } 78 | } 79 | 80 | keys := make([]string, 0, len(qemuCount)) 81 | for k := range qemuCount { 82 | keys = append(keys, k) 83 | } 84 | 85 | sort.SliceStable(keys, func(i, j int) bool { 86 | return qemuCount[keys[i]] < qemuCount[keys[j]] 87 | }) 88 | 89 | for _, cluster := range keys { 90 | if node, err := client.Cluster(cluster).GetQemuPlacableNode(request); err == nil && node != "" { 91 | if vmId, err := client.Cluster(cluster).GetNextId(); err == nil { 92 | var result QemuPlace 93 | 94 | result.Cluster = cluster 95 | result.Node = node 96 | result.VmId = vmId 97 | 98 | return result, nil 99 | } 100 | } 101 | 102 | } 103 | 104 | return QemuPlace{}, fmt.Errorf("cannot find available cluster") 105 | } 106 | 107 | func (client *Client) GetQemuPlace(name string) (QemuPlace, error) { 108 | var place QemuPlace 109 | for cluster, _ := range client.Clusters { 110 | resources, err := client.Cluster(cluster).GetResources(RESOURCE_QEMU) 111 | if err != nil { 112 | return place, err 113 | } 114 | 115 | for _, resource := range resources { 116 | if resource.Name == name { 117 | place.Cluster = cluster 118 | place.Node = resource.Node 119 | place.VmId = resource.Vmid 120 | place.Found = true 121 | 122 | return place, nil 123 | } 124 | 125 | } 126 | } 127 | 128 | return place, nil 129 | } 130 | -------------------------------------------------------------------------------- /cmd/proxmox/cluster.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "strconv" 23 | "strings" 24 | 25 | "github.com/go-resty/resty/v2" 26 | log "github.com/sirupsen/logrus" 27 | "k8s.io/utils/strings/slices" 28 | ) 29 | 30 | const ( 31 | RESOURCE_QEMU = "qemu" 32 | RESOURCE_LXC = "lxc" 33 | RESOURCE_OPENVZ = "openvz" 34 | RESOURCE_STORAGE = "storage" 35 | RESOURCE_NODE = "node" 36 | RESOURCE_SDN = "sdn" 37 | RESOURCE_POOL = "pool" 38 | ) 39 | 40 | type ClusterApiConfig struct { 41 | ApiUrl string `yaml:"api_url"` 42 | ApiTokenId string `yaml:"api_token_id"` 43 | ApiTokenSecret string `yaml:"api_token_secret"` 44 | Pool string `yaml:"pool"` 45 | } 46 | 47 | type Cluster struct { 48 | name string 49 | apiCOnfig ClusterApiConfig 50 | resty *resty.Client 51 | } 52 | 53 | type NextIdResp struct { 54 | NextId string `json:"data"` 55 | } 56 | 57 | type ResourcesResp struct { 58 | Data []Resource `json:"data"` 59 | } 60 | 61 | type Resource struct { 62 | Maxdisk int64 `json:"maxdisk"` 63 | Netout int64 `json:"netout,omitempty"` 64 | ID string `json:"id"` 65 | Vmid int `json:"vmid,omitempty"` 66 | Type string `json:"type"` 67 | Mem int64 `json:"mem,omitempty"` 68 | Diskread int64 `json:"diskread,omitempty"` 69 | Maxmem int64 `json:"maxmem,omitempty"` 70 | Template int `json:"template,omitempty"` 71 | Tags string `json:"tags,omitempty"` 72 | Status string `json:"status"` 73 | Netin int64 `json:"netin,omitempty"` 74 | Maxcpu int `json:"maxcpu,omitempty"` 75 | Node string `json:"node"` 76 | Uptime int `json:"uptime,omitempty"` 77 | Diskwrite int64 `json:"diskwrite,omitempty"` 78 | Name string `json:"name,omitempty"` 79 | CPU float64 `json:"cpu,omitempty"` 80 | Disk int `json:"disk"` 81 | Level string `json:"level,omitempty"` 82 | Shared int `json:"shared,omitempty"` 83 | Content string `json:"content,omitempty"` 84 | Storage string `json:"storage,omitempty"` 85 | Plugintype string `json:"plugintype,omitempty"` 86 | } 87 | 88 | type NodesResp struct { 89 | Nodes []NodeResp `json:"data"` 90 | } 91 | 92 | type NodeResp struct { 93 | Maxmem int64 `json:"maxmem"` 94 | Maxdisk int64 `json:"maxdisk"` 95 | ID string `json:"id"` 96 | Type string `json:"type"` 97 | Mem int64 `json:"mem"` 98 | Uptime int `json:"uptime"` 99 | SslFingerprint string `json:"ssl_fingerprint"` 100 | CPU float64 `json:"cpu"` 101 | Level string `json:"level"` 102 | Disk int64 `json:"disk"` 103 | Status string `json:"status"` 104 | Maxcpu int `json:"maxcpu"` 105 | Node string `json:"node"` 106 | } 107 | 108 | type PlaceRequest struct { 109 | Name string 110 | CPU int 111 | Mem int64 112 | AntiAffinity string 113 | } 114 | 115 | func (cluster *Cluster) GetReq(apiPath string, data interface{}) ([]byte, error) { 116 | resp, err := cluster.resty.R(). 117 | SetBody(data). 118 | SetHeader("Accept", "application/json"). 119 | SetHeader("Content-Type", "application/json"). 120 | Get(fmt.Sprintf("%s/%s", cluster.apiCOnfig.ApiUrl, apiPath)) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | if resp.IsError() { 126 | return nil, fmt.Errorf("proxmox api error: %d %s", resp.StatusCode(), resp.Body()) 127 | } 128 | 129 | return resp.Body(), nil 130 | } 131 | 132 | func (cluster *Cluster) PostReq(apiPath string, data interface{}) error { 133 | resp, err := cluster.resty.R(). 134 | SetBody(data). 135 | SetHeader("Accept", "application/json"). 136 | SetHeader("Content-Type", "application/json"). 137 | Post(fmt.Sprintf("%s/%s", cluster.apiCOnfig.ApiUrl, apiPath)) 138 | if err != nil { 139 | return err 140 | } 141 | 142 | if resp.IsError() { 143 | return fmt.Errorf("proxmox api error: %d %s %s", resp.StatusCode(), resp.Status(), resp.Body()) 144 | } 145 | 146 | return nil 147 | } 148 | 149 | func (cluster *Cluster) PutReq(apiPath string, data interface{}) error { 150 | resp, err := cluster.resty.R(). 151 | SetBody(data). 152 | SetHeader("Accept", "application/json"). 153 | SetHeader("Content-Type", "application/json"). 154 | Put(fmt.Sprintf("%s/%s", cluster.apiCOnfig.ApiUrl, apiPath)) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | if resp.IsError() { 160 | return fmt.Errorf("proxmox api error: %d %s %s", resp.StatusCode(), resp.Status(), resp.Body()) 161 | } 162 | 163 | return nil 164 | } 165 | 166 | func (cluster *Cluster) DeleteReq(apiPath string) error { 167 | resp, err := cluster.resty.R(). 168 | Delete(fmt.Sprintf("%s/%s", cluster.apiCOnfig.ApiUrl, apiPath)) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | if resp.IsError() { 174 | return fmt.Errorf("proxmox api error: %d %s", resp.StatusCode(), resp.Body()) 175 | } 176 | 177 | return nil 178 | } 179 | 180 | func (cluster *Cluster) GetNextId() (int, error) { 181 | log.Debugf("Get next id, cluster: %s", cluster.name) 182 | 183 | apiPath := "/cluster/nextid" 184 | 185 | data, err := cluster.GetReq(apiPath, nil) 186 | if err != nil { 187 | return -1, err 188 | } 189 | 190 | nextIdResp := NextIdResp{} 191 | err = json.Unmarshal(data, &nextIdResp) 192 | if err != nil { 193 | return -1, err 194 | } 195 | 196 | nextId, err := strconv.Atoi(nextIdResp.NextId) 197 | if err != nil { 198 | return -1, err 199 | } 200 | 201 | return nextId, err 202 | } 203 | 204 | func (cluster *Cluster) GetResources(resourceType string) ([]Resource, error) { 205 | apiPath := "/cluster/resources" 206 | 207 | reqData := fmt.Sprintf(`{"type":"%s"}`, resourceType) 208 | data, err := cluster.GetReq(apiPath, reqData) 209 | if err != nil { 210 | return nil, err 211 | } 212 | 213 | resourcesResp := ResourcesResp{} 214 | err = json.Unmarshal(data, &resourcesResp) 215 | if err != nil { 216 | return nil, err 217 | } 218 | 219 | return resourcesResp.Data, err 220 | } 221 | 222 | func (cluster *Cluster) Node(node string) *Node { 223 | result := Node{ 224 | name: node, 225 | cluster: *cluster, 226 | } 227 | 228 | return &result 229 | } 230 | 231 | func (cluster *Cluster) GetNodes() ([]NodeResp, error) { 232 | apiPath := "/nodes" 233 | 234 | data, err := cluster.GetReq(apiPath, nil) 235 | if err != nil { 236 | return nil, err 237 | } 238 | 239 | nodesData := NodesResp{} 240 | err = json.Unmarshal(data, &nodesData) 241 | if err != nil { 242 | return nil, err 243 | } 244 | 245 | return nodesData.Nodes, err 246 | } 247 | 248 | func (cluster *Cluster) GetNode(nodeName string) (NodeResp, error) { 249 | apiPath := "/nodes" 250 | 251 | data, err := cluster.GetReq(apiPath, nil) 252 | if err != nil { 253 | return NodeResp{}, err 254 | } 255 | 256 | nodesData := NodesResp{} 257 | err = json.Unmarshal(data, &nodesData) 258 | if err != nil { 259 | return NodeResp{}, err 260 | } 261 | 262 | for _, node := range nodesData.Nodes { 263 | if node.Node == nodeName { 264 | return node, nil 265 | } 266 | } 267 | 268 | return NodeResp{}, err 269 | } 270 | 271 | func (cluster *Cluster) GetResourceCount(resourceType string) (int, error) { 272 | resources, err := cluster.GetResources(resourceType) 273 | if err != nil { 274 | return -1, err 275 | } 276 | 277 | var result int 278 | for _, r := range resources { 279 | if r.Type == resourceType { 280 | result++ 281 | } 282 | } 283 | 284 | return result, nil 285 | } 286 | 287 | func (cluster *Cluster) GetQemuPlacableNode(request PlaceRequest) (string, error) { 288 | nodes, err := cluster.GetNodes() 289 | if err != nil { 290 | return "", err 291 | } 292 | 293 | var candidateNode string 294 | var prevCount int 295 | for _, node := range nodes { 296 | resources, err := cluster.Node(node.Node).GetResources(RESOURCE_QEMU) 297 | if err != nil { 298 | return "", err 299 | } 300 | 301 | var ignoreNode bool 302 | for _, resource := range resources { 303 | tags := strings.Split(resource.Tags, ";") 304 | if slices.Contains(tags, fmt.Sprintf("anti-affinity.%s", request.AntiAffinity)) { 305 | ignoreNode = true 306 | } 307 | } 308 | 309 | if ignoreNode { 310 | continue 311 | } 312 | 313 | qemuCount := len(resources) 314 | if placable, err := cluster.Node(node.Node).IsQemuPlacable(request.CPU, request.Mem); err == nil && placable { 315 | if candidateNode == "" || qemuCount < prevCount { 316 | candidateNode = node.Node 317 | prevCount = qemuCount 318 | } 319 | } 320 | } 321 | 322 | if candidateNode != "" { 323 | return candidateNode, nil 324 | } else { 325 | return "", fmt.Errorf("cannot find available node") 326 | } 327 | } 328 | 329 | func (cluster *Cluster) FindQemuPlace(name string) (QemuPlace, error) { 330 | var place QemuPlace 331 | 332 | nodes, err := cluster.GetNodes() 333 | if err != nil { 334 | return place, err 335 | } 336 | 337 | for _, node := range nodes { 338 | resources, err := cluster.Node(node.Node).GetResources(RESOURCE_QEMU) 339 | if err != nil { 340 | return place, err 341 | } 342 | 343 | for _, resource := range resources { 344 | if resource.Name == name { 345 | place.Found = true 346 | place.Cluster = cluster.name 347 | place.Node = node.Node 348 | place.VmId = resource.Vmid 349 | 350 | return place, nil 351 | } 352 | } 353 | } 354 | 355 | return place, fmt.Errorf("cannot find qemu place: %s", name) 356 | } 357 | -------------------------------------------------------------------------------- /cmd/proxmox/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | type Node struct { 20 | name string 21 | cluster Cluster 22 | } 23 | 24 | func (node *Node) Qemu() *Qemu { 25 | result := Qemu{ 26 | node: node, 27 | } 28 | 29 | return &result 30 | } 31 | 32 | func (node *Node) GetResources(resourceType string) ([]Resource, error) { 33 | resources, err := node.cluster.GetResources(resourceType) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | var result []Resource 39 | for _, resource := range resources { 40 | if resource.Node == node.name { 41 | result = append(result, resource) 42 | } 43 | } 44 | 45 | return result, nil 46 | } 47 | 48 | func (node *Node) GetResourceCount(resourceType string) (int, error) { 49 | resources, err := node.cluster.GetResources(resourceType) 50 | if err != nil { 51 | return -1, err 52 | } 53 | 54 | var result int 55 | for _, resource := range resources { 56 | if resource.Node == node.name && resource.Type == resourceType { 57 | result++ 58 | } 59 | } 60 | 61 | return result, nil 62 | } 63 | 64 | func (node *Node) IsQemuPlacable(cpu int, mem int64) (bool, error) { 65 | nodeResources, err := node.cluster.GetNode(node.name) 66 | if err != nil { 67 | return false, err 68 | } 69 | 70 | if (float64(nodeResources.Maxcpu)-nodeResources.CPU) > float64(cpu) && (nodeResources.Maxmem-nodeResources.Mem > mem) { 71 | return true, nil 72 | } 73 | 74 | return false, nil 75 | } 76 | -------------------------------------------------------------------------------- /cmd/proxmox/qemu.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | 23 | log "github.com/sirupsen/logrus" 24 | ) 25 | 26 | const ( 27 | STATUS_RUNNING = "running" 28 | STATUS_STOPPED = "stopped" 29 | ) 30 | 31 | type Qemu struct { 32 | node *Node 33 | } 34 | 35 | type QemuConfigResp struct { 36 | Data QemuConfig `json:"data"` 37 | } 38 | 39 | type ( 40 | QemuConfig map[string]interface{} 41 | ) 42 | 43 | type QemuPendingConfigResp struct { 44 | Data []QemuPendingConfig `json:"data"` 45 | } 46 | 47 | type QemuPendingConfig struct { 48 | Key string `json:"key"` 49 | Value interface{} `json:"value"` 50 | Pending interface{} `json:"pending"` 51 | } 52 | 53 | type QemuStatus struct { 54 | Data struct { 55 | Maxdisk int64 `json:"maxdisk"` 56 | Diskread int `json:"diskread"` 57 | Maxmem int64 `json:"maxmem"` 58 | CPU float64 `json:"cpu"` 59 | BalloonMin int64 `json:"balloon_min"` 60 | Disk int `json:"disk"` 61 | Qmpstatus string `json:"qmpstatus"` 62 | Uptime int `json:"uptime"` 63 | Netin int `json:"netin"` 64 | Shares int `json:"shares"` 65 | Ha struct { 66 | Managed int `json:"managed"` 67 | } `json:"ha"` 68 | Diskwrite int `json:"diskwrite"` 69 | Vmid int `json:"vmid"` 70 | Mem int `json:"mem"` 71 | Status string `json:"status"` 72 | Netout int `json:"netout"` 73 | Cpus int `json:"cpus"` 74 | Name string `json:"name"` 75 | } `json:"data"` 76 | } 77 | 78 | func checkQemuConfig(qemuConfig QemuConfig) error { 79 | if _, isExist := qemuConfig["node"]; !isExist { 80 | return fmt.Errorf("no node name in qemu config") 81 | } 82 | if _, isExist := qemuConfig["vmid"]; !isExist { 83 | return fmt.Errorf("no vmid in qemu config") 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func (qemu *Qemu) Create(qemuConfig QemuConfig) error { 90 | log.Infof("Creating qemu VM, cluster: %s, node: %s config: %+v", qemu.node.cluster.name, qemu.node.name, qemuConfig) 91 | err := checkQemuConfig(qemuConfig) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | apiPath := fmt.Sprintf("/nodes/%s/qemu", qemu.node.name) 97 | err = qemu.node.cluster.PostReq(apiPath, qemuConfig) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | return nil 103 | } 104 | 105 | func (qemu *Qemu) Clone(name string, templatePlace, targetPlace QemuPlace) error { 106 | log.Infof("Cloning qemu VM, cluster: %s, node: %s vmid: %d -> cluster: %s, node: %s vmid: %d name: %s", 107 | templatePlace.Cluster, 108 | templatePlace.Node, 109 | templatePlace.VmId, 110 | targetPlace.Cluster, 111 | targetPlace.Node, 112 | targetPlace.VmId, 113 | name, 114 | ) 115 | 116 | data := fmt.Sprintf(`{"vmid":"%d", "newid":"%d", "name":"%s", "target":"%s", "full":true}`, templatePlace.VmId, targetPlace.VmId, name, targetPlace.Node) 117 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/clone", templatePlace.Node, templatePlace.VmId) 118 | err := qemu.node.cluster.PostReq(apiPath, data) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | return nil 124 | } 125 | 126 | func (qemu *Qemu) SetConfig(qemuConfig QemuConfig) error { 127 | log.Infof("Set qemu VM config, cluster: %s, node: %s config: %+v", qemu.node.cluster.name, qemu.node.name, qemuConfig) 128 | err := checkQemuConfig(qemuConfig) 129 | if err != nil { 130 | return err 131 | } 132 | 133 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/config", qemu.node.name, qemuConfig["vmid"]) 134 | err = qemu.node.cluster.PostReq(apiPath, qemuConfig) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | return nil 140 | } 141 | 142 | func (qemu *Qemu) GetConfig(vmId int) (QemuConfig, error) { 143 | log.Debugf("Get qemu config, cluster: %s node: %s vmid: %d", qemu.node.cluster.name, qemu.node.name, vmId) 144 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/config", qemu.node.name, vmId) 145 | data := fmt.Sprintf(`{"node":"%s", "vmid":"%d"}`, qemu.node.name, vmId) 146 | 147 | qemuConfigData, err := qemu.node.cluster.GetReq(apiPath, data) 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | qemuConfig := QemuConfigResp{} 153 | err = json.Unmarshal(qemuConfigData, &qemuConfig) 154 | if err != nil { 155 | return nil, err 156 | } 157 | 158 | log.Debug(qemuConfig.Data) 159 | return qemuConfig.Data, nil 160 | } 161 | 162 | func (qemu *Qemu) GetPendingConfig(vmId int) ([]QemuPendingConfig, error) { 163 | log.Debugf("Get qemu pending config, cluster: %s node: %s vmid: %d", qemu.node.cluster.name, qemu.node.name, vmId) 164 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/pending", qemu.node.name, vmId) 165 | data := fmt.Sprintf(`{"node":"%s", "vmid":"%d"}`, qemu.node.name, vmId) 166 | 167 | qemuConfigData, err := qemu.node.cluster.GetReq(apiPath, data) 168 | if err != nil { 169 | return []QemuPendingConfig{}, err 170 | } 171 | 172 | pendingConfig := QemuPendingConfigResp{} 173 | err = json.Unmarshal(qemuConfigData, &pendingConfig) 174 | if err != nil { 175 | return []QemuPendingConfig{}, err 176 | } 177 | 178 | log.Debug(pendingConfig.Data) 179 | return pendingConfig.Data, nil 180 | } 181 | 182 | func (qemu *Qemu) Delete(vmId int) error { 183 | log.Infof("Deleting qemu VM, cluster: %s node: %s vmid: %d", qemu.node.cluster.name, qemu.node.name, vmId) 184 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d", qemu.node.name, vmId) 185 | err := qemu.node.cluster.DeleteReq(apiPath) 186 | if err != nil { 187 | return err 188 | } 189 | 190 | return nil 191 | } 192 | 193 | func (qemu *Qemu) Start(vmId int) error { 194 | log.Infof("Starting qemu, cluster: %s node: %s vmid: %d", qemu.node.cluster.name, qemu.node.name, vmId) 195 | data := fmt.Sprintf(`{"node":"%s", "vmid":"%d"}`, qemu.node.name, vmId) 196 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/status/start", qemu.node.name, vmId) 197 | err := qemu.node.cluster.PostReq(apiPath, data) 198 | if err != nil { 199 | return err 200 | } 201 | 202 | return nil 203 | } 204 | 205 | func (qemu *Qemu) Stop(vmId int) error { 206 | log.Infof("Stopping qemu, cluster: %s node: %s vmid: %d", qemu.node.cluster.name, qemu.node.name, vmId) 207 | data := fmt.Sprintf(`{"node":"%s", "vmid":"%d"}`, qemu.node.name, vmId) 208 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/status/stop", qemu.node.name, vmId) 209 | err := qemu.node.cluster.PostReq(apiPath, data) 210 | if err != nil { 211 | return err 212 | } 213 | 214 | return nil 215 | } 216 | 217 | func (qemu *Qemu) GetStatus(vmId int) (QemuStatus, error) { 218 | log.Debugf("Get qemu status, cluster: %s node: %s vmid: %d", qemu.node.cluster.name, qemu.node.name, vmId) 219 | 220 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/status/current", qemu.node.name, vmId) 221 | data := fmt.Sprintf(`{"node":"%s", "vmid":"%d"}`, qemu.node.name, vmId) 222 | 223 | qemuStatusData, err := qemu.node.cluster.GetReq(apiPath, data) 224 | if err != nil { 225 | return QemuStatus{}, err 226 | } 227 | 228 | qemuStatus := QemuStatus{} 229 | err = json.Unmarshal(qemuStatusData, &qemuStatus) 230 | if err != nil { 231 | return QemuStatus{}, err 232 | } 233 | 234 | log.Debug(qemuStatus) 235 | return qemuStatus, nil 236 | } 237 | 238 | func (qemu *Qemu) Resize(vmId int, disk, size string) error { 239 | log.Infof("Resize qemu disk, cluster: %s node: %s vmid: %d disk: %s size: %s", qemu.node.cluster.name, qemu.node.name, vmId, disk, size) 240 | data := fmt.Sprintf(`{"node":"%s", "vmid":"%d", "disk":"%s", "size":"%s"}`, qemu.node.name, vmId, disk, size) 241 | apiPath := fmt.Sprintf("/nodes/%s/qemu/%d/resize", qemu.node.name, vmId) 242 | err := qemu.node.cluster.PutReq(apiPath, data) 243 | if err != nil { 244 | return err 245 | } 246 | 247 | return nil 248 | } 249 | -------------------------------------------------------------------------------- /cmd/proxmox/storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "fmt" 21 | 22 | log "github.com/sirupsen/logrus" 23 | ) 24 | 25 | type DiskConfig struct { 26 | Name string `json:"-"` 27 | Node string `json:"node"` 28 | VmId int `json:"vmid"` 29 | Filename string `json:"filename"` 30 | Size string `json:"size"` 31 | Storage string `json:"storage"` 32 | } 33 | 34 | func (node *Node) DiskCreate(diskConfig DiskConfig) error { 35 | log.Debugf("Creating disk, cluster: %s, node: %s config: %+v", node.cluster.name, node.name, diskConfig) 36 | apiPath := fmt.Sprintf("/nodes/%s/storage/%s/content", node.name, diskConfig.Storage) 37 | err := node.cluster.PostReq(apiPath, diskConfig) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | listen: :8080 2 | clusters: 3 | pve-test: 4 | api_token_id: root@pam!proxmox-operator 5 | api_token_secret: XXX 6 | api_url: https://10.171.120.253:8006/api2/json 7 | pool: infra 8 | log: 9 | format: text 10 | level: info 11 | -------------------------------------------------------------------------------- /examples/example-qemu.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: proxmox.xfix.org/v1alpha1 2 | kind: Qemu 3 | metadata: 4 | name: example-qemu 5 | finalizers: 6 | - resources-finalizer.proxmox-operator.xfix.org 7 | spec: 8 | cluster: pve-test 9 | #node: crash-lab ### If not set it will set automaticly from "pool" 10 | #vmid: 222 ### If not set it will set automaticly 11 | pool: prod ### Cluster pool for place VM 12 | anti-affinity: "" ### The anti-affinity group. VM's with same anti-affinity group will be placed on different nodes 13 | autostart: true 14 | autostop: true 15 | cpu: 16 | type: host 17 | sockets: 2 18 | cores: 1 19 | memory: 20 | size: 2048 21 | balloon: 2048 22 | network: 23 | net0: 24 | model: virtio 25 | #mac: A2:7B:45:48:9C:E6 ### If not set it will set automaticly 26 | bridge: vmbr0 27 | tag: 103 # set to 0 if tag no need 28 | disk: 29 | scsi0: 30 | storage: local-lvm 31 | size: 9G 32 | tags: 33 | - test1 34 | - test2 35 | options: 36 | ostype: "l26" 37 | bios: "seabios" 38 | smbios1: "uuid=3ae878b3-a77e-4a4a-adc6-14ee88350d36,manufacturer=MTIz,product=MTIz,version=MTIz,serial=MTIz,sku=MTIz,family=MTIz,base64=1" 39 | scsihw: "virtio-scsi-pci" 40 | boot: "order=net0;ide2;scsi0" 41 | ide2: "none,media=cdrom" 42 | hotplug: "network,disk,usb" 43 | tablet: 1 44 | onboot: 0 45 | kvm: 1 46 | agent: "0" 47 | numa: 1 48 | protection: 0 49 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/CRASH-Tech/proxmox-operator 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/go-resty/resty/v2 v2.7.0 7 | github.com/prometheus/client_golang v1.17.0 8 | github.com/sirupsen/logrus v1.9.0 9 | golang.org/x/exp v0.0.0-20230206171751-46f607a40771 10 | gopkg.in/yaml.v2 v2.4.0 11 | k8s.io/apimachinery v0.26.0 12 | k8s.io/client-go v0.26.0 13 | k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 14 | ) 15 | 16 | require ( 17 | github.com/beorn7/perks v1.0.1 // indirect 18 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 21 | github.com/go-logr/logr v1.2.3 // indirect 22 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 23 | github.com/go-openapi/jsonreference v0.20.0 // indirect 24 | github.com/go-openapi/swag v0.19.14 // indirect 25 | github.com/gogo/protobuf v1.3.2 // indirect 26 | github.com/golang/protobuf v1.5.3 // indirect 27 | github.com/google/gnostic v0.5.7-v3refs // indirect 28 | github.com/google/go-cmp v0.5.9 // indirect 29 | github.com/google/gofuzz v1.1.0 // indirect 30 | github.com/imdario/mergo v0.3.6 // indirect 31 | github.com/josharian/intern v1.0.0 // indirect 32 | github.com/json-iterator/go v1.1.12 // indirect 33 | github.com/mailru/easyjson v0.7.6 // indirect 34 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 35 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 36 | github.com/modern-go/reflect2 v1.0.2 // indirect 37 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 38 | github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect 39 | github.com/prometheus/common v0.44.0 // indirect 40 | github.com/prometheus/procfs v0.11.1 // indirect 41 | github.com/spf13/pflag v1.0.5 // indirect 42 | golang.org/x/net v0.10.0 // indirect 43 | golang.org/x/oauth2 v0.8.0 // indirect 44 | golang.org/x/sys v0.11.0 // indirect 45 | golang.org/x/term v0.8.0 // indirect 46 | golang.org/x/text v0.9.0 // indirect 47 | golang.org/x/time v0.3.0 // indirect 48 | google.golang.org/appengine v1.6.7 // indirect 49 | google.golang.org/protobuf v1.31.0 // indirect 50 | gopkg.in/inf.v0 v0.9.1 // indirect 51 | gopkg.in/yaml.v3 v3.0.1 // indirect 52 | k8s.io/api v0.26.0 // indirect 53 | k8s.io/klog/v2 v2.80.1 // indirect 54 | k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect 55 | sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect 56 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 57 | sigs.k8s.io/yaml v1.3.0 // indirect 58 | ) 59 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 6 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 7 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 8 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 9 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 10 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 12 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= 14 | github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= 15 | github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 16 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 17 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 18 | github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= 19 | github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 20 | github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= 21 | github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 22 | github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 23 | github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= 24 | github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 25 | github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= 26 | github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= 27 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 28 | github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= 29 | github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= 30 | github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= 31 | github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= 32 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 33 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 34 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 35 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 36 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 37 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 38 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 39 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 40 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 41 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 42 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 43 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 44 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 45 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 46 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 47 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 48 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 49 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 50 | github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= 51 | github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= 52 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 53 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 54 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 55 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 56 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 57 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 58 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 59 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 60 | github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= 61 | github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 62 | github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= 63 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 64 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 65 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 66 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 67 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 68 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 69 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 70 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 71 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 72 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 73 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 74 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 75 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 76 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 77 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 78 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 79 | github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= 80 | github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 81 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 82 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 83 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 84 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 85 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 86 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 87 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 88 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 89 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 90 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 91 | github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= 92 | github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= 93 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 94 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 95 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 96 | github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= 97 | github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= 98 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 99 | github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= 100 | github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= 101 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 102 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 103 | github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= 104 | github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= 105 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 106 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 107 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 108 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 109 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 110 | github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= 111 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 112 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 113 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 114 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 115 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 116 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 117 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 118 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 119 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 120 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 121 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 122 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 123 | golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= 124 | golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= 125 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 126 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 127 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 128 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 129 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 130 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 131 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 132 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 133 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 134 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 135 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 136 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 137 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 138 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 139 | golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 140 | golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= 141 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 142 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 143 | golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= 144 | golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= 145 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 146 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 147 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 148 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 149 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 150 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 151 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 152 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 153 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 154 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 155 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 156 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 158 | golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= 159 | golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 160 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 161 | golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= 162 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 163 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 164 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 165 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 166 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 167 | golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= 168 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 169 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 170 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 171 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 172 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 173 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 174 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 175 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 176 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 177 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 178 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 179 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 180 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 181 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 182 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 183 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 184 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 185 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 186 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 187 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 188 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 189 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 190 | google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 191 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 192 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 193 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 194 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 195 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 196 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 197 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 198 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 199 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 200 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 201 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 202 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 203 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 204 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 205 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 206 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 207 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 208 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 209 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 210 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 211 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 212 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 213 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 214 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 215 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 216 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 217 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 218 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 219 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 220 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 221 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 222 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 223 | k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= 224 | k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= 225 | k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= 226 | k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= 227 | k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= 228 | k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= 229 | k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= 230 | k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= 231 | k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= 232 | k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= 233 | k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= 234 | k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 235 | sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= 236 | sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 237 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= 238 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= 239 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 240 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 241 | -------------------------------------------------------------------------------- /leaderElection.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "time" 7 | 8 | log "github.com/sirupsen/logrus" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/client-go/tools/leaderelection" 11 | "k8s.io/client-go/tools/leaderelection/resourcelock" 12 | ) 13 | 14 | func getNewLock(lockname, podname, namespace string) *resourcelock.LeaseLock { 15 | return &resourcelock.LeaseLock{ 16 | LeaseMeta: metav1.ObjectMeta{ 17 | Name: lockname, 18 | Namespace: namespace, 19 | }, 20 | Client: config.KubernetesClient.CoordinationV1(), 21 | LockConfig: resourcelock.ResourceLockConfig{ 22 | Identity: podname, 23 | }, 24 | } 25 | } 26 | 27 | func setLeaderLabel(isLeader bool) { 28 | pod, err := config.KubernetesClient.CoreV1().Pods(namespace).Get(context.TODO(), hostname, metav1.GetOptions{}) 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | 33 | existingLabels := pod.ObjectMeta.Labels 34 | if isLeader { 35 | existingLabels["leader"] = "true" 36 | } else { 37 | existingLabels["leader"] = "false" 38 | } 39 | 40 | pod.ObjectMeta.Labels = existingLabels 41 | 42 | updatedPod, err := config.KubernetesClient.CoreV1().Pods(namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | 47 | log.Infof("Updated pod label: %s\n", updatedPod.Labels["leader"]) 48 | } 49 | 50 | func runLeaderElection(lock *resourcelock.LeaseLock, ctx context.Context, id string) { 51 | leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ 52 | Lock: lock, 53 | ReleaseOnCancel: true, 54 | LeaseDuration: 15 * time.Second, 55 | RenewDeadline: 10 * time.Second, 56 | RetryPeriod: 2 * time.Second, 57 | Callbacks: leaderelection.LeaderCallbacks{ 58 | OnStartedLeading: func(c context.Context) { 59 | mutex.Unlock() 60 | setLeaderLabel(true) 61 | worker() 62 | }, 63 | OnStoppedLeading: func() { 64 | log.Warn("We are no longer the leader, terminating...") 65 | mutex.Lock() 66 | setLeaderLabel(false) 67 | 68 | os.Exit(0) 69 | }, 70 | OnNewLeader: func(current_id string) { 71 | if current_id == id { 72 | log.Info("We are still the leading!") 73 | 74 | return 75 | } 76 | log.Warnf("New leader is %s", current_id) 77 | }, 78 | }, 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "flag" 23 | "fmt" 24 | "io/ioutil" 25 | "net/http" 26 | "os" 27 | "regexp" 28 | "strconv" 29 | "strings" 30 | "sync" 31 | "time" 32 | 33 | "github.com/CRASH-Tech/proxmox-operator/cmd/common" 34 | kubernetes "github.com/CRASH-Tech/proxmox-operator/cmd/kubernetes" 35 | "github.com/CRASH-Tech/proxmox-operator/cmd/kubernetes/api/v1alpha1" 36 | "github.com/CRASH-Tech/proxmox-operator/cmd/proxmox" 37 | "github.com/prometheus/client_golang/prometheus" 38 | "github.com/prometheus/client_golang/prometheus/promhttp" 39 | log "github.com/sirupsen/logrus" 40 | "golang.org/x/exp/slices" 41 | "k8s.io/client-go/dynamic" 42 | k8s "k8s.io/client-go/kubernetes" 43 | "k8s.io/client-go/rest" 44 | "k8s.io/client-go/tools/clientcmd" 45 | ) 46 | 47 | var ( 48 | version = "0.1.5" 49 | config common.Config 50 | kClient *kubernetes.Client 51 | pClient *proxmox.Client 52 | namespace string 53 | hostname string 54 | leaseLockName = "proxmox-operator" 55 | mutex sync.Mutex 56 | 57 | qemuStatus = prometheus.NewGaugeVec( 58 | prometheus.GaugeOpts{ 59 | Name: "qemu_status", 60 | Help: "The qemu status", 61 | }, 62 | []string{ 63 | "cluster", 64 | "node", 65 | "vmid", 66 | "vmname", 67 | }, 68 | ) 69 | 70 | qemuPower = prometheus.NewGaugeVec( 71 | prometheus.GaugeOpts{ 72 | Name: "qemu_power", 73 | Help: "The qemu power status", 74 | }, 75 | []string{ 76 | "cluster", 77 | "node", 78 | "vmid", 79 | "vmname", 80 | }, 81 | ) 82 | ) 83 | 84 | func init() { 85 | var configPath string 86 | flag.StringVar(&configPath, "c", "config.yaml", "config file path. Default: config.yaml") 87 | c, err := readConfig(configPath) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | config = c 92 | 93 | switch config.Log.Format { 94 | case "text": 95 | log.SetFormatter(&log.TextFormatter{}) 96 | case "json": 97 | log.SetFormatter(&log.JSONFormatter{}) 98 | default: 99 | log.SetFormatter(&log.TextFormatter{}) 100 | } 101 | 102 | switch config.Log.Level { 103 | case "info": 104 | log.SetLevel(log.InfoLevel) 105 | case "warn": 106 | log.SetLevel(log.WarnLevel) 107 | case "debug": 108 | log.SetLevel(log.DebugLevel) 109 | default: 110 | log.SetLevel(log.InfoLevel) 111 | } 112 | 113 | var restConfig *rest.Config 114 | if path, isSet := os.LookupEnv("KUBECONFIG"); isSet { 115 | log.Printf("Using configuration from '%s'", path) 116 | restConfig, err = clientcmd.BuildConfigFromFlags("", path) 117 | if err != nil { 118 | log.Fatal(err) 119 | } 120 | } else { 121 | log.Printf("Using in-cluster configuration") 122 | restConfig, err = rest.InClusterConfig() 123 | if err != nil { 124 | log.Fatal(err) 125 | } 126 | } 127 | 128 | config.DynamicClient = dynamic.NewForConfigOrDie(restConfig) 129 | config.KubernetesClient = k8s.NewForConfigOrDie(restConfig) 130 | 131 | ns, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") 132 | if err != nil { 133 | log.Panic(err) 134 | } 135 | 136 | namespace = string(ns) 137 | hostname = os.Getenv("HOSTNAME") 138 | 139 | prometheus.MustRegister(qemuStatus) 140 | prometheus.MustRegister(qemuPower) 141 | } 142 | 143 | func main() { 144 | ctx, cancel := context.WithCancel(context.Background()) 145 | defer cancel() 146 | 147 | kClient = kubernetes.NewClient(ctx, *config.DynamicClient) 148 | pClient = proxmox.NewClient(config.Clusters) 149 | 150 | mutex.Lock() 151 | setLeaderLabel(false) 152 | 153 | lock := getNewLock(leaseLockName, hostname, namespace) 154 | runLeaderElection(lock, ctx, hostname) 155 | 156 | } 157 | 158 | func worker() { 159 | log.Infof("Starting proxmox-operator %s", version) 160 | 161 | listen() 162 | 163 | for { 164 | metrics() 165 | processV1aplha1(kClient, pClient) 166 | 167 | time.Sleep(5 * time.Second) 168 | } 169 | } 170 | 171 | func listen() { 172 | go func() { 173 | http.Handle("/metrics", promhttp.Handler()) 174 | err := http.ListenAndServe(config.Listen, nil) 175 | if err != nil { 176 | log.Panic(err) 177 | } 178 | }() 179 | } 180 | 181 | func metrics() { 182 | qemus, err := kClient.V1alpha1().Qemu().GetAll() 183 | if err != nil { 184 | log.Error(err) 185 | 186 | return 187 | } 188 | 189 | for _, qemu := range qemus { 190 | if qemu.Status.VmId == 0 { 191 | continue 192 | } 193 | 194 | var status float64 195 | if qemu.Status.Status == "SYNCED" { 196 | status = 1 197 | } else { 198 | status = -1 199 | } 200 | qemuStatus.WithLabelValues( 201 | qemu.Status.Cluster, 202 | qemu.Status.Node, 203 | strconv.Itoa(qemu.Status.VmId), 204 | qemu.Metadata.Name, 205 | ).Set(status) 206 | 207 | var power float64 208 | if qemu.Status.Power == "ON" { 209 | power = 1 210 | } else { 211 | power = -1 212 | } 213 | qemuPower.WithLabelValues( 214 | qemu.Status.Cluster, 215 | qemu.Status.Node, 216 | strconv.Itoa(qemu.Status.VmId), 217 | qemu.Metadata.Name, 218 | ).Set(power) 219 | } 220 | } 221 | 222 | func processV1aplha1(kClient *kubernetes.Client, pClient *proxmox.Client) { 223 | mutex.Lock() 224 | defer mutex.Unlock() 225 | 226 | log.Info("Refreshing v1alpha1...") 227 | qemus, err := kClient.V1alpha1().Qemu().GetAll() 228 | if err != nil { 229 | log.Error(err) 230 | return 231 | } 232 | 233 | for _, qemu := range qemus { 234 | switch qemu.Status.Status { 235 | case v1alpha1.STATUS_QEMU_DELETING: 236 | qemu, err := deleteQemu(pClient, qemu) 237 | if err != nil { 238 | log.Errorf("cannot delete qemu %s: %s", qemu.Metadata.Name, err) 239 | 240 | continue 241 | } 242 | 243 | qemu.RemoveFinalizers() 244 | _, err = kClient.V1alpha1().Qemu().Patch(qemu) 245 | if err != nil { 246 | log.Errorf("cannot patch qemu cr %s: %s", qemu.Metadata.Name, err) 247 | 248 | continue 249 | } 250 | 251 | continue 252 | case v1alpha1.STATUS_QEMU_EMPTY: 253 | if qemu.Status.Status == v1alpha1.STATUS_QEMU_EMPTY && qemu.Metadata.DeletionTimestamp != "" { 254 | qemu.RemoveFinalizers() 255 | _, err = kClient.V1alpha1().Qemu().Patch(qemu) 256 | if err != nil { 257 | log.Errorf("cannot patch qemu cr %s: %s", qemu.Metadata.Name, err) 258 | 259 | continue 260 | } 261 | 262 | continue 263 | } 264 | 265 | if qemu.Spec.Clone != "" { 266 | qemu.Status.Status = v1alpha1.STATUS_QEMU_CLONING 267 | qemu, err = updateQemuStatus(kClient, qemu) 268 | if err != nil { 269 | return 270 | } 271 | 272 | continue 273 | } 274 | 275 | qemu, err := getQemuPlace(pClient, qemu) 276 | if err != nil { 277 | log.Errorf("cannot get qemu place %s: %s", qemu.Metadata.Name, err) 278 | 279 | continue 280 | } 281 | 282 | if qemu.Status.Status == v1alpha1.STATUS_QEMU_OUT_OF_SYNC { 283 | qemu, err = updateQemuStatus(kClient, qemu) 284 | if err != nil { 285 | return 286 | } 287 | 288 | continue 289 | } 290 | 291 | qemu, err = createNewQemu(pClient, qemu) 292 | if err != nil { 293 | log.Errorf("cannot create qemu %s: %s", qemu.Metadata.Name, err) 294 | if qemu.Status.Status == v1alpha1.STATUS_QEMU_EMPTY { 295 | qemu = cleanQemuPlaceStatus(qemu) 296 | } 297 | 298 | qemu, err = updateQemuStatus(kClient, qemu) 299 | if err != nil { 300 | return 301 | } 302 | 303 | continue 304 | } 305 | 306 | qemu.Status.Status = v1alpha1.STATUS_QEMU_SYNCED 307 | qemu, err = updateQemuStatus(kClient, qemu) 308 | if err != nil { 309 | return 310 | } 311 | 312 | // Need by proxmox api delay 313 | time.Sleep(time.Second * 10) 314 | 315 | continue 316 | case v1alpha1.STATUS_QEMU_CLONING: 317 | if qemu.Spec.Cluster == "" { 318 | log.Error("no cluster set for clone operation") 319 | 320 | continue 321 | } 322 | 323 | templatePlace, err := pClient.Cluster(qemu.Spec.Cluster).FindQemuPlace(qemu.Spec.Clone) 324 | if err != nil { 325 | log.Error(err) 326 | 327 | continue 328 | } 329 | 330 | qemu, err = getQemuPlace(pClient, qemu) 331 | if err != nil { 332 | log.Error(err) 333 | 334 | continue 335 | } 336 | 337 | targetPlace := proxmox.QemuPlace{ 338 | Found: true, 339 | Cluster: qemu.Status.Cluster, 340 | Node: qemu.Status.Node, 341 | VmId: qemu.Status.VmId, 342 | } 343 | 344 | err = pClient.Cluster(templatePlace.Cluster).Node(templatePlace.Node).Qemu().Clone(qemu.Metadata.Name, templatePlace, targetPlace) 345 | if err != nil { 346 | log.Error(err) 347 | qemu.Status.Status = v1alpha1.STATUS_QEMU_EMPTY 348 | qemu, err = updateQemuStatus(kClient, qemu) 349 | if err != nil { 350 | return 351 | } 352 | continue 353 | } 354 | 355 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 356 | qemu, err = updateQemuStatus(kClient, qemu) 357 | if err != nil { 358 | return 359 | } 360 | 361 | qemu, err = setQemuConfig(pClient, qemu) 362 | if err != nil { 363 | log.Errorf("cannot set qemu config %s: %s", qemu.Metadata.Name, err) 364 | continue 365 | } 366 | 367 | qemu.Status.Status = v1alpha1.STATUS_QEMU_SYNCED 368 | qemu, err = updateQemuStatus(kClient, qemu) 369 | if err != nil { 370 | return 371 | } 372 | 373 | // Need by proxmox api delay 374 | time.Sleep(time.Second * 10) 375 | 376 | continue 377 | case v1alpha1.STATUS_QEMU_SYNCED, 378 | v1alpha1.STATUS_QEMU_OUT_OF_SYNC, 379 | v1alpha1.STATUS_QEMU_PENDING, 380 | v1alpha1.STATUS_QEMU_UNKNOWN: 381 | place, err := pClient.GetQemuPlace(qemu.Metadata.Name) 382 | if err != nil { 383 | log.Errorf("cannot get qemu place %s: %s", qemu.Metadata.Name, err) 384 | qemu.Status.Status = v1alpha1.STATUS_QEMU_UNKNOWN 385 | qemu = cleanQemuPlaceStatus(qemu) 386 | qemu, err = updateQemuStatus(kClient, qemu) 387 | if err != nil { 388 | return 389 | } 390 | 391 | continue 392 | } 393 | 394 | if !place.Found { 395 | qemu.Status.Status = v1alpha1.STATUS_QEMU_UNKNOWN 396 | qemu = cleanQemuPlaceStatus(qemu) 397 | qemu, err = updateQemuStatus(kClient, qemu) 398 | if err != nil { 399 | return 400 | } 401 | 402 | continue 403 | } else { 404 | qemu = updateQemuPlaceStatus(place, qemu) 405 | qemu, err = updateQemuStatus(kClient, qemu) 406 | if err != nil { 407 | return 408 | } 409 | 410 | qemu, err = getQemuPowerStatus(pClient, qemu) 411 | if err != nil { 412 | log.Errorf("cannot get qemu power status %s: %s", qemu.Metadata.Name, err) 413 | qemu.Status.Status = v1alpha1.STATUS_QEMU_UNKNOWN 414 | qemu, err = updateQemuStatus(kClient, qemu) 415 | if err != nil { 416 | return 417 | } 418 | 419 | continue 420 | } 421 | 422 | qemu, err = getQemuNetStatus(pClient, qemu) 423 | if err != nil { 424 | log.Errorf("cannot get qemu network status %s: %s", qemu.Metadata.Name, err) 425 | qemu.Status.Status = v1alpha1.STATUS_QEMU_UNKNOWN 426 | qemu, err = updateQemuStatus(kClient, qemu) 427 | if err != nil { 428 | return 429 | } 430 | 431 | continue 432 | } 433 | 434 | qemu, err = updateQemuStatus(kClient, qemu) 435 | if err != nil { 436 | return 437 | } 438 | } 439 | 440 | switch qemu.Status.Status { 441 | case v1alpha1.STATUS_QEMU_OUT_OF_SYNC, v1alpha1.STATUS_QEMU_PENDING: 442 | qemu, err := setQemuConfig(pClient, qemu) 443 | if err != nil { 444 | log.Errorf("cannot set qemu config %s: %s", qemu.Metadata.Name, err) 445 | } 446 | 447 | qemu, err = updateQemuStatus(kClient, qemu) 448 | if err != nil { 449 | return 450 | } 451 | 452 | continue 453 | default: 454 | qemu, err = checkQemuSyncStatus(pClient, qemu) 455 | if err != nil { 456 | log.Errorf("cannot get qemu sync status %s: %s", qemu.Metadata.Name, err) 457 | qemu.Status.Status = v1alpha1.STATUS_QEMU_UNKNOWN 458 | qemu, err = updateQemuStatus(kClient, qemu) 459 | if err != nil { 460 | return 461 | } 462 | 463 | continue 464 | } 465 | 466 | qemu, err = updateQemuStatus(kClient, qemu) 467 | if err != nil { 468 | return 469 | } 470 | 471 | continue 472 | } 473 | default: 474 | log.Warnf("unknown qemu state: %s %s", qemu.Metadata.Name, qemu.Status.Status) 475 | 476 | continue 477 | } 478 | } 479 | 480 | } 481 | 482 | func updateQemuPlaceStatus(place proxmox.QemuPlace, qemu v1alpha1.Qemu) v1alpha1.Qemu { 483 | qemu.Status.Cluster = place.Cluster 484 | qemu.Status.Node = place.Node 485 | qemu.Status.VmId = place.VmId 486 | 487 | return qemu 488 | } 489 | 490 | func cleanQemuPlaceStatus(qemu v1alpha1.Qemu) v1alpha1.Qemu { 491 | qemu.Status.Cluster = "" 492 | qemu.Status.Node = "" 493 | qemu.Status.VmId = 0 494 | 495 | return qemu 496 | } 497 | 498 | func updateQemuStatus(kClient *kubernetes.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 499 | name := qemu.Metadata.Name 500 | qemu, err := kClient.V1alpha1().Qemu().UpdateStatus(qemu) 501 | if err != nil { 502 | return qemu, fmt.Errorf("cannot update qemu status %s: %s", name, err) 503 | } 504 | return qemu, nil 505 | } 506 | 507 | func getQemuPlace(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 508 | placeRequest := buildPlaceRequest(qemu) 509 | place, err := pClient.GetQemuPlace(qemu.Metadata.Name) 510 | if err != nil { 511 | return qemu, fmt.Errorf("cannot check is qemu already exist: %s", err) 512 | } 513 | 514 | if place.Found { 515 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 516 | qemu.Status.Cluster = place.Cluster 517 | qemu.Status.Node = place.Node 518 | qemu.Status.VmId = place.VmId 519 | 520 | return qemu, nil 521 | } 522 | 523 | if qemu.Spec.Cluster == "" && qemu.Spec.Pool == "" { 524 | return qemu, fmt.Errorf("no cluster or pool are set for: %s", qemu.Metadata.Name) 525 | } 526 | 527 | if qemu.Spec.Pool != "" { 528 | var err error 529 | place, err = pClient.GetQemuPlacableCluster(placeRequest) 530 | if err != nil { 531 | return qemu, fmt.Errorf("cannot find autoplace cluster: %s", err) 532 | } 533 | } 534 | 535 | if qemu.Spec.Cluster == "" { 536 | qemu.Status.Cluster = place.Cluster 537 | } else { 538 | qemu.Status.Cluster = qemu.Spec.Cluster 539 | } 540 | 541 | if qemu.Spec.Node == "" { 542 | node, err := pClient.Cluster(qemu.Status.Cluster).GetQemuPlacableNode(placeRequest) 543 | if err != nil { 544 | return qemu, fmt.Errorf("cannot find available node: %s", err) 545 | } 546 | qemu.Status.Node = node 547 | } else { 548 | qemu.Status.Node = qemu.Spec.Node 549 | } 550 | 551 | if qemu.Spec.VmId == 0 { 552 | nextId, err := pClient.Cluster(qemu.Status.Cluster).GetNextId() 553 | if err != nil { 554 | return qemu, fmt.Errorf("cannot get qemu next id: %s", err) 555 | } 556 | qemu.Status.VmId = nextId 557 | } else { 558 | qemu.Status.VmId = qemu.Spec.VmId 559 | } 560 | 561 | return qemu, nil 562 | } 563 | 564 | func createNewQemu(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 565 | qemuConfig, err := buildQemuConfig(pClient, qemu) 566 | if err != nil { 567 | qemu.Status.Status = v1alpha1.STATUS_QEMU_EMPTY 568 | return qemu, fmt.Errorf("cannot build qemu config: %s", err) 569 | } 570 | 571 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().Create(qemuConfig) 572 | if err != nil { 573 | qemu.Status.Status = v1alpha1.STATUS_QEMU_EMPTY 574 | return qemu, fmt.Errorf("cannot create qemu: %s", err) 575 | } 576 | 577 | qemuConfig, err = createQemuDisks(pClient, qemu, qemuConfig) 578 | if err != nil { 579 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 580 | return qemu, fmt.Errorf("cannot create qemu disk: %s", err) 581 | } 582 | 583 | pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().SetConfig(qemuConfig) 584 | if err != nil { 585 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 586 | return qemu, fmt.Errorf("cannot set qemu config: %s", err) 587 | } 588 | 589 | if qemu.Spec.Autostart { 590 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().Start(qemu.Status.VmId) 591 | if err != nil { 592 | log.Errorf("cannot start qemu: %s %s", qemu.Metadata.Name, err) 593 | return qemu, nil 594 | } 595 | } 596 | 597 | return qemu, nil 598 | } 599 | 600 | func buildQemuConfig(client *proxmox.Client, qemu v1alpha1.Qemu) (proxmox.QemuConfig, error) { 601 | result := make(map[string]interface{}) 602 | 603 | result["vmid"] = qemu.Status.VmId 604 | result["node"] = qemu.Status.Node 605 | result["name"] = qemu.Metadata.Name 606 | result["cpu"] = qemu.Spec.CPU.Type 607 | result["sockets"] = qemu.Spec.CPU.Sockets 608 | result["cores"] = qemu.Spec.CPU.Cores 609 | result["memory"] = qemu.Spec.Memory.Size 610 | result["balloon"] = qemu.Spec.Memory.Balloon 611 | 612 | ifaceCurrentMacs := make(map[string]string) 613 | for _, data := range qemu.Status.Net { 614 | ifaceCurrentMacs[data.Name] = data.Mac 615 | } 616 | 617 | for ifaceName, iface := range qemu.Spec.Network { 618 | if iface.Mac == "" { 619 | if ifaceCurrentMacs[ifaceName] == "" { 620 | if iface.Tag == 0 { 621 | result[ifaceName] = fmt.Sprintf("%s,bridge=%s", iface.Model, iface.Bridge) 622 | } else { 623 | result[ifaceName] = fmt.Sprintf("%s,bridge=%s,tag=%d", iface.Model, iface.Bridge, iface.Tag) 624 | } 625 | } else { 626 | if iface.Tag == 0 { 627 | result[ifaceName] = fmt.Sprintf("%s=%s,bridge=%s", iface.Model, ifaceCurrentMacs[ifaceName], iface.Bridge) 628 | } else { 629 | result[ifaceName] = fmt.Sprintf("%s=%s,bridge=%s,tag=%d", iface.Model, ifaceCurrentMacs[ifaceName], iface.Bridge, iface.Tag) 630 | } 631 | } 632 | } else { 633 | if iface.Tag == 0 { 634 | result[ifaceName] = fmt.Sprintf("%s=%s,bridge=%s", iface.Model, iface.Mac, iface.Bridge) 635 | } else { 636 | result[ifaceName] = fmt.Sprintf("%s=%s,bridge=%s,tag=%d", iface.Model, iface.Mac, iface.Bridge, iface.Tag) 637 | } 638 | } 639 | } 640 | 641 | for k, v := range qemu.Spec.Options { 642 | result[k] = v 643 | } 644 | 645 | var tags []string 646 | tags = append(tags, qemu.Spec.Tags...) 647 | 648 | if qemu.Spec.AntiAffinity != "" { 649 | tags = append(tags, fmt.Sprintf("anti-affinity.%s", qemu.Spec.AntiAffinity)) 650 | } 651 | 652 | if len(tags) > 0 { 653 | result["tags"] = strings.Join(tags, ";") 654 | } 655 | 656 | return result, nil 657 | } 658 | 659 | func createQemuDisks(pClient *proxmox.Client, qemu v1alpha1.Qemu, qemuConfig proxmox.QemuConfig) (proxmox.QemuConfig, error) { 660 | disksConfig, err := buildDisksConfig(qemu) 661 | if err != nil { 662 | return qemuConfig, fmt.Errorf("cannot build disks config: %s", err) 663 | } 664 | 665 | for _, diskConfig := range disksConfig { 666 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).DiskCreate(diskConfig) 667 | if err != nil { 668 | return qemuConfig, fmt.Errorf("cannot create qemu disk: %s", err) 669 | } 670 | qemuConfig[diskConfig.Name] = fmt.Sprintf("%s:%s,size=%s", diskConfig.Storage, diskConfig.Filename, diskConfig.Size) 671 | } 672 | 673 | return qemuConfig, nil 674 | } 675 | 676 | func buildDisksConfig(qemu v1alpha1.Qemu) ([]proxmox.DiskConfig, error) { 677 | var disksConfig []proxmox.DiskConfig 678 | 679 | for diskName, disk := range qemu.Spec.Disk { 680 | diskConfig, err := buildDiskConfig(qemu, diskName, disk) 681 | if err != nil { 682 | return disksConfig, fmt.Errorf("cannot build disk config: %s %s", disk, err) 683 | } 684 | 685 | disksConfig = append(disksConfig, diskConfig) 686 | } 687 | return disksConfig, nil 688 | } 689 | 690 | func buildDiskConfig(qemu v1alpha1.Qemu, diskName string, disk v1alpha1.QemuDisk) (proxmox.DiskConfig, error) { 691 | var diskConfig proxmox.DiskConfig 692 | rDiskNum := regexp.MustCompile(`^[a-z]+(\d+)$`) 693 | diskNum := rDiskNum.FindStringSubmatch(diskName) 694 | if len(diskNum) != 2 { 695 | return diskConfig, fmt.Errorf("cannot extract disk num: %s", diskName) 696 | } 697 | 698 | filename := fmt.Sprintf("vm-%d-disk-%s", qemu.Status.VmId, diskNum[1]) 699 | diskConfig = proxmox.DiskConfig{ 700 | Name: diskName, 701 | Node: qemu.Status.Node, 702 | VmId: qemu.Status.VmId, 703 | Filename: filename, 704 | Size: disk.Size, 705 | Storage: disk.Storage, 706 | } 707 | 708 | return diskConfig, nil 709 | } 710 | 711 | func buildPlaceRequest(qemu v1alpha1.Qemu) proxmox.PlaceRequest { 712 | var result proxmox.PlaceRequest 713 | 714 | result.Name = qemu.Metadata.Name 715 | result.CPU = qemu.Spec.CPU.Sockets + qemu.Spec.CPU.Cores 716 | result.Mem = qemu.Spec.Memory.Size 717 | result.AntiAffinity = qemu.Spec.AntiAffinity 718 | 719 | return result 720 | } 721 | 722 | func getQemuPowerStatus(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 723 | qemuStatus, err := pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().GetStatus(qemu.Status.VmId) 724 | if err != nil { 725 | qemu.Status.Power = v1alpha1.STATUS_POWER_UNKNOWN 726 | return qemu, fmt.Errorf("cannot get qemu power status: %s", err) 727 | } 728 | 729 | switch qemuStatus.Data.Status { 730 | case proxmox.STATUS_RUNNING: 731 | qemu.Status.Power = v1alpha1.STATUS_POWER_ON 732 | case proxmox.STATUS_STOPPED: 733 | qemu.Status.Power = v1alpha1.STATUS_POWER_OFF 734 | default: 735 | qemu.Status.Power = v1alpha1.STATUS_POWER_UNKNOWN 736 | } 737 | 738 | return qemu, nil 739 | } 740 | 741 | func getQemuNetStatus(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 742 | currentConfig, err := pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().GetConfig(qemu.Status.VmId) 743 | if err != nil { 744 | return qemu, fmt.Errorf("cannot get qemu config from proxmox: %s", err) 745 | } 746 | 747 | var ifacesConfig []v1alpha1.QemuStatusNetwork 748 | rExp := "(.{2}:.{2}:.{2}:.{2}:.{2}:.{2})" 749 | r := regexp.MustCompile(rExp) 750 | for ifaceName, _ := range qemu.Spec.Network { 751 | var ifaceConfig v1alpha1.QemuStatusNetwork 752 | ifaceConfig.Name = ifaceName 753 | macData := r.FindStringSubmatch(fmt.Sprintf("%s", currentConfig[ifaceName])) 754 | if len(macData) > 0 { 755 | ifaceConfig.Mac = macData[0] 756 | } 757 | 758 | ifacesConfig = append(ifacesConfig, ifaceConfig) 759 | } 760 | qemu.Status.Net = ifacesConfig 761 | 762 | return qemu, nil 763 | } 764 | 765 | func deleteQemu(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 766 | if qemu.Status.Cluster == "" || qemu.Status.Node == "" || qemu.Status.VmId == 0 { 767 | return qemu, fmt.Errorf("unknown qemu status") 768 | } 769 | 770 | if qemu.Spec.Autostop { 771 | err := pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().Stop(qemu.Status.VmId) 772 | if err != nil { 773 | return qemu, fmt.Errorf("cannot stop qemu: %s", err) 774 | } 775 | } 776 | 777 | qemu, err := getQemuPowerStatus(pClient, qemu) 778 | if err != nil { 779 | return qemu, fmt.Errorf("cannot get qemu power status: %s", err) 780 | } 781 | if qemu.Status.Power == v1alpha1.STATUS_POWER_ON { 782 | return qemu, errors.New("waiting for qemu stop") 783 | } else { 784 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().Delete(qemu.Status.VmId) 785 | if err != nil { 786 | return qemu, fmt.Errorf("cannot delete qemu: %s", err) 787 | } 788 | } 789 | 790 | return qemu, nil 791 | } 792 | 793 | func checkQemuSyncStatus(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 794 | //Check deleting state 795 | if qemu.Metadata.DeletionTimestamp != "" { 796 | qemu.Status.Status = v1alpha1.STATUS_QEMU_DELETING 797 | 798 | return qemu, nil 799 | } 800 | 801 | //Check config state 802 | currentConfig, err := pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().GetConfig(qemu.Status.VmId) 803 | if err != nil { 804 | return qemu, fmt.Errorf("cannot get qemu current config: %s", err) 805 | } 806 | 807 | designConfig, err := buildQemuConfig(pClient, qemu) 808 | if err != nil { 809 | return qemu, fmt.Errorf("cannot build qemu config: %s", err) 810 | } 811 | 812 | var outOfSync bool 813 | for k, v := range designConfig { 814 | v := strings.TrimRight(fmt.Sprint(v), "\n") // FUCK YOU PROXMOX! 815 | 816 | if k == "node" || k == "vmid" { 817 | continue 818 | } 819 | 820 | if k == "tags" { 821 | designTags := strings.Split(v, ";") 822 | currentTags := strings.Split(fmt.Sprint(currentConfig["tags"]), ";") 823 | 824 | for _, tag := range designTags { 825 | if !slices.Contains(currentTags, tag) { 826 | log.Warnf("Qemu %s is out of sync, tag %s is not found", qemu.Metadata.Name, tag) 827 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 828 | return qemu, nil 829 | } 830 | } 831 | 832 | continue 833 | } 834 | 835 | if fmt.Sprint(currentConfig[k]) != v { 836 | log.Warnf("Qemu %s is out of sync, %s: %s != %s", qemu.Metadata.Name, k, currentConfig[k], v) 837 | outOfSync = true 838 | } 839 | } 840 | 841 | if outOfSync { 842 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 843 | return qemu, nil 844 | } 845 | 846 | pendingConfig, err := pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().GetPendingConfig(qemu.Status.VmId) 847 | if err != nil { 848 | return qemu, fmt.Errorf("cannot get pending config: %s", err) 849 | } 850 | 851 | //Check Disks config 852 | rDiskSize := regexp.MustCompile(`^.+size=(.+),?$`) 853 | for diskName, disk := range qemu.Spec.Disk { 854 | var isDiskFound bool 855 | var isDiskChanged bool 856 | for param, paramValue := range currentConfig { 857 | if diskName == param { 858 | isDiskFound = true 859 | currentSize := rDiskSize.FindStringSubmatch(fmt.Sprint(paramValue)) 860 | if len(currentSize) != 2 { 861 | return qemu, fmt.Errorf("cannot parse disk params: %s %s", param, paramValue) 862 | } 863 | if disk.Size != currentSize[1] { 864 | isDiskChanged = true 865 | } 866 | } 867 | } 868 | if !isDiskFound || isDiskChanged { 869 | log.Warnf("Qemu %s disk is not found or changed: %s", qemu.Metadata.Name, disk) 870 | qemu.Status.Status = v1alpha1.STATUS_QEMU_OUT_OF_SYNC 871 | 872 | return qemu, nil 873 | } 874 | } 875 | 876 | //Check pending state 877 | var isPending bool 878 | for _, v := range pendingConfig { 879 | if v.Pending != nil && v.Value != v.Pending { 880 | log.Warnf("Qemu %s is in pending state, %s: %v != %v", qemu.Metadata.Name, v.Key, v.Value, v.Pending) 881 | isPending = true 882 | } 883 | } 884 | 885 | if isPending { 886 | qemu.Status.Status = v1alpha1.STATUS_QEMU_PENDING 887 | return qemu, nil 888 | } 889 | 890 | qemu.Status.Status = v1alpha1.STATUS_QEMU_SYNCED 891 | 892 | return qemu, nil 893 | } 894 | 895 | func setQemuConfig(pClient *proxmox.Client, qemu v1alpha1.Qemu) (v1alpha1.Qemu, error) { 896 | qemuConfig, err := buildQemuConfig(pClient, qemu) 897 | if err != nil { 898 | return qemu, fmt.Errorf("cannot build qemu config: %s", err) 899 | } 900 | 901 | currentConfig, err := pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().GetConfig(qemu.Status.VmId) 902 | if err != nil { 903 | return qemu, fmt.Errorf("cannot get qemu current config: %s", err) 904 | } 905 | rDiskSize := regexp.MustCompile(`^.+size=(.+),?$`) 906 | for diskName, disk := range qemu.Spec.Disk { 907 | var isDiskFound bool 908 | for param, paramValue := range currentConfig { 909 | if diskName == param { 910 | isDiskFound = true 911 | currentSize := rDiskSize.FindStringSubmatch(fmt.Sprint(paramValue)) 912 | if len(currentSize) != 2 { 913 | return qemu, fmt.Errorf("cannot parse disk params: %s %s", param, paramValue) 914 | } 915 | if disk.Size != currentSize[1] { 916 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().Resize(qemu.Status.VmId, diskName, disk.Size) 917 | if err != nil { 918 | return qemu, fmt.Errorf("cannot resize qemu disk: %s", err) 919 | } 920 | } 921 | } 922 | } 923 | if !isDiskFound { 924 | log.Warnf("Qemu %s disk is not found: %s", qemu.Metadata.Name, disk) 925 | diskConfig, err := buildDiskConfig(qemu, diskName, disk) 926 | if err != nil { 927 | return qemu, err 928 | } 929 | 930 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).DiskCreate(diskConfig) 931 | if err != nil { 932 | log.Errorf("cannot create qemu disk: %s", err) 933 | } 934 | qemuConfig[diskName] = fmt.Sprintf("%s:%s,size=%s", diskConfig.Storage, diskConfig.Filename, diskConfig.Size) 935 | 936 | } 937 | } 938 | 939 | err = pClient.Cluster(qemu.Status.Cluster).Node(qemu.Status.Node).Qemu().SetConfig(qemuConfig) 940 | if err != nil { 941 | return qemu, fmt.Errorf("cannot set qemu config: %s", err) 942 | } 943 | 944 | qemu, err = checkQemuSyncStatus(pClient, qemu) 945 | if err != nil { 946 | qemu.Status.Status = v1alpha1.STATUS_QEMU_UNKNOWN 947 | } 948 | 949 | return qemu, nil 950 | } 951 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The CRASH-Tech. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "io/ioutil" 21 | 22 | "github.com/CRASH-Tech/proxmox-operator/cmd/common" 23 | "github.com/CRASH-Tech/proxmox-operator/cmd/proxmox" 24 | "gopkg.in/yaml.v2" 25 | ) 26 | 27 | func readConfig(path string) (common.Config, error) { 28 | config := common.Config{} 29 | config.Clusters = make(map[string]proxmox.ClusterApiConfig) 30 | 31 | yamlFile, err := ioutil.ReadFile(path) 32 | if err != nil { 33 | return common.Config{}, err 34 | } 35 | err = yaml.Unmarshal(yamlFile, &config) 36 | if err != nil { 37 | return common.Config{}, err 38 | } 39 | 40 | return config, err 41 | } 42 | --------------------------------------------------------------------------------