├── .dockerignore ├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── .go-version ├── .golangci.yml ├── Dockerfile ├── Makefile ├── PROJECT ├── README.md ├── api └── v1beta1 │ ├── conditions_consts.go │ ├── consts.go │ ├── elfcluster_types.go │ ├── elfmachine_types.go │ ├── elfmachinetemplate_types.go │ ├── groupversion_info.go │ ├── types.go │ └── zz_generated.deepcopy.go ├── clusterctl-settings.json ├── config ├── certmanager │ ├── certificate.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── crd │ ├── bases │ │ ├── infrastructure.cluster.x-k8s.io_elfclusters.yaml │ │ ├── infrastructure.cluster.x-k8s.io_elfmachines.yaml │ │ ├── infrastructure.cluster.x-k8s.io_elfmachinetemplates.yaml │ │ ├── kubesmart.smtx.io_hostconfigs.yaml │ │ └── kubesmart.smtx.io_hostoperationjobs.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_elfclusters.yaml │ │ ├── cainjection_in_elfmachines.yaml │ │ ├── cainjection_in_elfmachinetemplates.yaml │ │ ├── webhook_in_elfclusters.yaml │ │ ├── webhook_in_elfmachines.yaml │ │ └── webhook_in_elfmachinetemplates.yaml ├── default │ ├── kustomization.yaml │ ├── manager_config_patch.yaml │ ├── manager_image_patch.yaml │ ├── manager_pull_policy.yaml │ ├── manager_webhook_patch.yaml │ ├── namespace.yaml │ └── webhookcainjection_patch.yaml ├── manager │ ├── controller_manager_config.yaml │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── elfcluster_editor_role.yaml │ ├── elfcluster_viewer_role.yaml │ ├── elfmachine_editor_role.yaml │ ├── elfmachine_viewer_role.yaml │ ├── elfmachinetemplate_editor_role.yaml │ ├── elfmachinetemplate_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ ├── role_binding.yaml │ └── service_account.yaml ├── samples │ ├── infrastructure_v1alpha3_elfcluster.yaml │ ├── infrastructure_v1alpha3_elfmachine.yaml │ └── infrastructure_v1alpha3_elfmachinetemplate.yaml └── webhook │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ ├── manifests.yaml │ └── service.yaml ├── controllers ├── elfcluster_controller.go ├── elfcluster_controller_test.go ├── elfmachine_controller.go ├── elfmachine_controller_gpu.go ├── elfmachine_controller_gpu_test.go ├── elfmachine_controller_placement_group.go ├── elfmachine_controller_resources.go ├── elfmachine_controller_resources_test.go ├── elfmachine_controller_test.go ├── suite_test.go ├── tower_cache.go ├── tower_cache_test.go ├── vm_limiter.go └── vm_limiter_test.go ├── docs └── releasing.md ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt ├── fetch_ext_bins.sh └── version.sh ├── main.go ├── metadata.yaml ├── pkg ├── config │ ├── config.go │ └── vm.go ├── constants │ └── constants.go ├── context │ ├── cluster_context.go │ ├── controller_manager_context.go │ ├── machine_context.go │ └── machine_template_context.go ├── errors │ └── consts.go ├── hostagent │ ├── service.go │ └── tasks │ │ ├── expand_root_partition.yaml │ │ ├── restart_kubelet.yaml │ │ └── tasks.go ├── manager │ ├── constants.go │ ├── manager.go │ └── options.go ├── product │ └── product.go ├── resources │ ├── label.go │ ├── placement_group.go │ └── resource.go ├── service │ ├── collections.go │ ├── collections_test.go │ ├── consts.go │ ├── errors.go │ ├── mock_services │ │ └── vm_mock.go │ ├── types.go │ ├── util.go │ ├── util_test.go │ └── vm.go ├── session │ ├── tower.go │ └── tower_test.go ├── util │ ├── annotations │ │ └── helpers.go │ ├── common.go │ ├── kcp │ │ └── kcp.go │ ├── kube.go │ ├── labels │ │ ├── helpers.go │ │ └── helpers_test.go │ ├── machine │ │ ├── kcp.go │ │ ├── kcp_test.go │ │ ├── machine.go │ │ ├── machine_test.go │ │ ├── md.go │ │ └── md_test.go │ ├── md │ │ ├── md.go │ │ └── md_test.go │ ├── patch │ │ └── finalizer.go │ ├── tower.go │ └── types │ │ ├── uuid.go │ │ └── uuid_test.go └── version │ ├── consts.go │ ├── util.go │ ├── util_test.go │ └── version.go ├── templates └── cluster-template.yaml ├── test ├── config │ └── host-agent │ │ ├── kubesmart.smtx.io_hostconfigs.yaml │ │ └── kubesmart.smtx.io_hostoperationjobs.yaml ├── e2e │ ├── README.md │ ├── cape_quick_start_test.go │ ├── cluster_upgrade.go │ ├── cluster_upgrade_test.go │ ├── common.go │ ├── config │ │ └── elf-dev.yaml │ ├── controlplane_helpers.go │ ├── daemonset_helpers.go │ ├── data │ │ ├── cape │ │ │ └── metadata.yaml │ │ ├── capi │ │ │ └── metadata.yaml │ │ ├── cni │ │ │ └── calico │ │ │ │ └── calico.yaml │ │ └── infrastructure-elf │ │ │ └── kustomization │ │ │ ├── base │ │ │ ├── cluster-resource-set-label.yaml │ │ │ ├── cluster-resource-set.yaml │ │ │ └── kustomization.yaml │ │ │ ├── cluster-template-cp-ha │ │ │ └── kustomization.yaml │ │ │ ├── cluster-template-kcp-remediation │ │ │ ├── kustomization.yaml │ │ │ └── mhc.yaml │ │ │ ├── cluster-template-kcp-scale-in │ │ │ ├── kcp-surge.yaml │ │ │ └── kustomization.yaml │ │ │ ├── cluster-template-md-remediation │ │ │ ├── kustomization.yaml │ │ │ ├── md.yaml │ │ │ └── mhc.yaml │ │ │ ├── cluster-template-node-drain │ │ │ ├── kcp-drain.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── md-drain.yaml │ │ │ └── vcpu.yaml │ │ │ └── conformance │ │ │ ├── kustomization.yaml │ │ │ ├── worker-node-md.yaml │ │ │ └── worker-node-size.yaml │ ├── e2e_suite_test.go │ ├── ha_test.go │ ├── k8s_conformance_test.go │ ├── kcp_scale_test.go │ ├── machinedeployment_helpers.go │ ├── md_remediations_test.go │ ├── md_rollout_test.go │ ├── md_scale_test.go │ ├── node_drain_timeout_test.go │ ├── node_helpers.go │ ├── tower_test.go │ └── vm_helpers.go ├── fake │ ├── controller_manager_context.go │ ├── tower.go │ └── types.go └── helpers │ ├── cluster.go │ ├── envtest.go │ ├── framework.go │ ├── mod.go │ └── mod_test.go ├── tools └── cert-manager │ └── cert-manager.yaml └── webhooks ├── elfmachine_webhook_mutation.go ├── elfmachine_webhook_mutation_test.go ├── elfmachine_webhook_validation.go ├── elfmachine_webhook_validation_test.go ├── elfmachinetemplate_webhook_mutation.go ├── elfmachinetemplate_webhook_mutation_test.go ├── elfmachinetemplate_webhook_validation.go ├── elfmachinetemplate_webhook_validation_test.go └── util.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore all files which are not go type 3 | !**/*.go 4 | !**/*.mod 5 | !**/*.sum 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - master 8 | - release-* 9 | tags: 10 | - v* 11 | pull_request: 12 | branches: 13 | - main 14 | - master 15 | - release-* 16 | 17 | jobs: 18 | build: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - name: Calculate go version 24 | run: echo "go_version=$(make go-version)" >> $GITHUB_ENV 25 | 26 | - name: Set up Go 27 | uses: actions/setup-go@v5 28 | with: 29 | go-version: ${{ env.go_version }} 30 | 31 | - run: make lint 32 | 33 | - run: make test 34 | 35 | - uses: codecov/codecov-action@v4 36 | with: 37 | token: ${{ secrets.CODECOV_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | # Sequence of patterns matched against refs/tags 6 | tags: 7 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 8 | 9 | jobs: 10 | release: 11 | name: Create draft release 12 | runs-on: ubuntu-latest 13 | environment: default 14 | steps: 15 | - name: Set environment variables 16 | run: | 17 | echo "IMAGE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV 18 | 19 | if [ "${{ secrets.REGISTRY }}" != "" ]; then 20 | echo "REGISTRY=${{ secrets.REGISTRY }}" >> $GITHUB_ENV 21 | fi 22 | 23 | - name: Checkout code 24 | uses: actions/checkout@v4 25 | 26 | - name: Calculate go version 27 | run: echo "go_version=$(make go-version)" >> $GITHUB_ENV 28 | 29 | - name: Set up Go 30 | uses: actions/setup-go@v5 31 | with: 32 | go-version: ${{ env.go_version }} 33 | 34 | - name: Login docker 35 | uses: docker/login-action@v3 36 | with: 37 | username: ${{ secrets.DOCKERHUB_USERNAME }} 38 | password: ${{ secrets.DOCKERHUB_TOKEN }} 39 | 40 | - name: Generate release 41 | run: make release 42 | 43 | - name: Generate draft release 44 | uses: softprops/action-gh-release@v2 45 | with: 46 | draft: true 47 | tag_name: ${{ env.TAG }} 48 | name: Release ${{ env.TAG }} 49 | files: out/* 50 | env: 51 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Test 2 | bin 3 | _artifacts 4 | test/e2e/data/infrastructure-elf/kustomization/base/cluster-template.yaml 5 | test/e2e/data/infrastructure-elf/cluster-template*.yaml 6 | 7 | # Editor 8 | .vscode 9 | .idea 10 | 11 | # Generated yaml in out/ 12 | out/ 13 | 14 | # Files for build 15 | .build 16 | 17 | 18 | # https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore 19 | 20 | # Binaries for programs and plugins 21 | *.exe 22 | *.exe~ 23 | *.dll 24 | *.so 25 | *.dylib 26 | 27 | # Test binary, built with `go test -c` 28 | *.test 29 | 30 | # Output of the go coverage tool, specifically when used with LiteIDE 31 | *.out 32 | 33 | # Dependency directories (remove the comment below to include it) 34 | # vendor/ 35 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.23.7 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1.4 2 | 3 | # Copyright 2022. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Build architecture 18 | ARG ARCH 19 | 20 | # Build the manager binary 21 | FROM golang:1.23.7 as builder 22 | WORKDIR /workspace 23 | 24 | # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy 25 | ARG goproxy=https://proxy.golang.org 26 | ENV GOPROXY=$goproxy 27 | 28 | # Copy the Go Modules manifests 29 | COPY go.mod go.mod 30 | COPY go.sum go.sum 31 | # Cache deps before building and copying source so that we don't need to re-download as much 32 | # and so that source changes don't invalidate our downloaded layer 33 | RUN go mod download 34 | 35 | # Copy the sources 36 | COPY ./ ./ 37 | 38 | # Build 39 | ARG ARCH 40 | ARG ldflags 41 | 42 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ 43 | go build -a -ldflags "${ldflags} -extldflags '-static'" \ 44 | -o manager . 45 | 46 | # Copy the controller-manager into a thin image 47 | FROM gcr.io/distroless/static:nonroot-${ARCH} 48 | WORKDIR / 49 | COPY --from=builder /workspace/manager . 50 | # Use uid of nonroot user (65532) because kubernetes expects numeric user when applying PSPs 51 | USER 65532 52 | ENTRYPOINT ["/manager"] 53 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Code generated by tool. DO NOT EDIT. 2 | # This file is used to track the info used to scaffold your project 3 | # and allow the plugins properly work. 4 | # More info: https://book.kubebuilder.io/reference/project-config.html 5 | domain: cluster.x-k8s.io 6 | layout: 7 | - go.kubebuilder.io/v3 8 | projectName: cluster-api-provider-elf 9 | repo: github.com/smartxworks/cluster-api-provider-elf 10 | resources: 11 | - api: 12 | crdVersion: v1 13 | namespaced: true 14 | controller: true 15 | domain: cluster.x-k8s.io 16 | group: infrastructure 17 | kind: ElfCluster 18 | path: github.com/smartxworks/cluster-api-provider-elf/api/v1beta1 19 | version: v1beta1 20 | - api: 21 | crdVersion: v1 22 | namespaced: true 23 | controller: true 24 | domain: cluster.x-k8s.io 25 | group: infrastructure 26 | kind: ElfMachine 27 | path: github.com/smartxworks/cluster-api-provider-elf/api/v1beta1 28 | version: v1beta1 29 | webhooks: 30 | defaulting: true 31 | validation: true 32 | webhookVersion: v1 33 | - api: 34 | crdVersion: v1 35 | namespaced: true 36 | controller: true 37 | domain: cluster.x-k8s.io 38 | group: infrastructure 39 | kind: ElfMachineTemplate 40 | path: github.com/smartxworks/cluster-api-provider-elf/api/v1beta1 41 | version: v1beta1 42 | webhooks: 43 | defaulting: true 44 | validation: true 45 | webhookVersion: v1 46 | version: "3" 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Cluster API Provider ELF 2 | 3 | [![build](https://github.com/smartxworks/cluster-api-provider-elf/actions/workflows/build.yml/badge.svg)](https://github.com/smartxworks/cluster-api-provider-elf/actions/workflows/build.yml) 4 | 5 | ## 准备K8s集群 6 | 7 | ### CAPI管理集群 8 | 9 | 首先创建一个普通的k8s集群。在开发环境中，可以使用kind,minikube等创建。参考[CAPI Quick Start Guide](https://cluster-api.sigs.k8s.io/user/quick-start.html). 10 | 11 | 在k8s基础集群中安装[cluster-api](https://github.com/kubernetes-sigs/cluster-api)后，将其变成CAPI管理集群(Management Cluster)。管理集群负责管理k8s工作负载集群(Workload Cluster)，包括创建、删除、扩容、升级等操作。 12 | 13 | ### 安装CAPE 14 | 15 | 安装Cluster API Provider ELF (CAPE): 16 | ``` 17 | export CONTROLLER_IMG=smartxworks/cape-manager-amd64 IMAGE_TAG=dev 18 | make deploy 19 | ``` 20 | 21 | ### 准备虚拟机模板 22 | 23 | 创建 k8s 集群时，先在 ELF 创建虚拟机，然后在虚拟机部署k8s节点。创建k8s集群，需要提供符合cluster-api要求的虚拟机。 24 | 可以使用 [image-builder](https://github.smartx.com/yiran/image-builder) 构建满足cluster-api要求的虚拟机模板，在创建k8s工作负载集群的时候使用指定的虚拟机模板。 25 | 26 | ## 使用方式 27 | 28 | **生成工作负载集群的配置文件** 29 | 30 | 提示：请根据​实际情况修改下述配置信息。 31 | 32 | ```shell 33 | # CAPE集群对象所在的namespace 34 | export NAMESPACE=default 35 | 36 | # 集群的名称 37 | export CLUSTER_NAME=cape-cluster 38 | 39 | # 集群的K8s版本 40 | export KUBERNETES_VERSION=v1.20.6 41 | 42 | # 集群ControlPlane节点数 43 | export CONTROL_PLANE_MACHINE_COUNT=1 44 | 45 | # 集群ControlPlane节点配置 46 | export CONTROL_PLANE_MACHINE_NUM_CPUS=2 47 | export CONTROL_PLANE_MACHINE_MEMORY_MB=4096 48 | export CONTROL_PLANE_MACHINE_DISK_GB=40 49 | 50 | # 集群Worker节点数 51 | export WORKER_MACHINE_COUNT=1 52 | 53 | # 集群Worker节点配置 54 | export WORKER_MACHINE_NUM_CPUS=2 55 | export WORKER_MACHINE_MEMORY_MB=4096 56 | export WORKER_MACHINE_DISK_GB=40 57 | 58 | # 集群节点的克隆模式: FastClone, FullClone. 59 | export ELF_VM_CLONE_MODE=FastClone 60 | 61 | # TOWER 62 | export TOWER_SERVER= 63 | export TOWER_USERNAME=root 64 | export TOWER_PASSWORD=tower 65 | export TOWER_SKIP_TLS_VERIFY=false 66 | # 认证模式，默认LOCAL（可选） 67 | export TOWER_AUTH_MODE=LOCAL 68 | 69 | # 以下指定的ELF资源对象的ID对应其API返回的local_id 70 | # ELF集群ID 71 | export ELF_CLUSTER=576ad467-d09e-4235-9dec-b615814ddc7e 72 | 73 | # ELF虚拟网络ID 74 | export ELF_VLAN=576ad467-d09e-4235-9dec-b615814ddc7e_c8a1e42d-e0f3-4d50-a190-53209a98f157 75 | 76 | # Control plane endpoint 77 | export CONTROL_PLANE_ENDPOINT_IP= 78 | 79 | # 内容库的虚拟机模板名或ID 80 | export VM_TEMPLATE=cl7hao0tseso80758osh921f1 81 | 82 | # 创建工作负载集群的配置文件 83 | clusterctl generate yaml --from templates/cluster-template.yaml > cape-cluster.yaml 84 | ``` 85 | 86 | **创建工作负载集群** 87 | 88 | ```shell 89 | kubectl apply -f cape-cluster.yaml 90 | ``` 91 | 92 | **检查集群部署情况** 93 | 94 | ```shell 95 | # 查看部署进度，直到INITIALIZED为true，表示ControlPlane已创建成功 96 | kubectl get cluster 97 | kubectl get kubeadmcontrolplane -w 98 | 99 | # 获取集群的 kubeconfig 配置 100 | clusterctl get kubeconfig cape-cluster > cape-cluster.kubeconfig 101 | 102 | # 部署 CNI, 可以选择 flannel 或者 calico（可选） 103 | kubectl --kubeconfig=./cape-cluster.kubeconfig \ 104 | apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml 105 | 106 | kubectl --kubeconfig=./cape-cluster.kubeconfig \ 107 | apply -f https://docs.projectcalico.org/v3.15/manifests/calico.yaml 108 | 109 | # 检查集群的节点 110 | KUBECONFIG=cape-cluster.kubeconfig kubectl get nodes 111 | 112 | # 检查集群的组件 113 | KUBECONFIG=cape-cluster.kubeconfig kubectl get pods -A -o wide 114 | ``` 115 | -------------------------------------------------------------------------------- /api/v1beta1/consts.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta1 18 | 19 | import ( 20 | "time" 21 | ) 22 | 23 | const ( 24 | // VMDisconnectionTimeout is the time allowed for the virtual machine to be disconnected. 25 | // The virtual machine will be marked as deleted after the timeout. 26 | VMDisconnectionTimeout = 1 * time.Minute 27 | ) 28 | 29 | // Annotations. 30 | const ( 31 | // PlacementGroupNameAnnotation is the annotation identifying the name of placement group. 32 | PlacementGroupNameAnnotation = "cape.infrastructure.cluster.x-k8s.io/placement-group-name" 33 | 34 | // CAPEVersionAnnotation is the annotation identifying the version of CAPE that the resource reconciled by. 35 | CAPEVersionAnnotation = "cape.infrastructure.cluster.x-k8s.io/cape-version" 36 | 37 | // CreatedByAnnotation is the annotation identifying the creator of the resource. 38 | // 39 | // The creator can be in one of the following two formats: 40 | // 1. ${Tower username}@${Tower auth_config_id}, e.g. caas.smartx@7e98ecbb-779e-43f6-8330-1bc1d29fffc7. 41 | // 2. ${Tower username}, e.g. root. If auth_config_id is not set, it means it is a LOCAL user. 42 | CreatedByAnnotation = "cape.infrastructure.cluster.x-k8s.io/created-by" 43 | ) 44 | 45 | // Labels. 46 | const ( 47 | // HostServerIDLabel is the label set on nodes. 48 | // It is the Tower ID of host server where the virtual machine runs on. 49 | HostServerIDLabel = "cape.infrastructure.cluster.x-k8s.io/host-server-id" 50 | 51 | // HostServerNameLabel is the label set on nodes. 52 | // It is the name of host server where the virtual machine runs on. 53 | HostServerNameLabel = "cape.infrastructure.cluster.x-k8s.io/host-server-name" 54 | 55 | // TowerVMIDLabel is the label set on nodes. 56 | // It is the Tower VM ID. 57 | TowerVMIDLabel = "cape.infrastructure.cluster.x-k8s.io/tower-vm-id" 58 | 59 | // NodeGroupLabel is the label set on nodes. 60 | // It is the node group name. 61 | // Node group name of CP node parses from KCP name, 62 | // and worker node parses from MD name. 63 | NodeGroupLabel = "cape.infrastructure.cluster.x-k8s.io/node-group" 64 | ) 65 | -------------------------------------------------------------------------------- /api/v1beta1/elfcluster_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 22 | ) 23 | 24 | const ( 25 | // ClusterFinalizer allows ReconcileElfCluster to clean up ELF 26 | // resources associated with ElfCluster before removing it from the 27 | // API server. 28 | ClusterFinalizer = "elfcluster.infrastructure.cluster.x-k8s.io" 29 | 30 | // ElfClusterForceDeleteAnnotation means to skip the deletion of infrastructure resources in Tower (e.g. VM and labels) 31 | // when deleting an ElfCluster. This is useful when the Tower server or SMTX ELF cluster is disconnected. 32 | ElfClusterForceDeleteAnnotation = "cape.infrastructure.cluster.x-k8s.io/force-delete-cluster" 33 | ) 34 | 35 | // ElfClusterSpec defines the desired state of ElfCluster. 36 | type ElfClusterSpec struct { 37 | // Cluster is a unique identifier for a ELF cluster. 38 | Cluster string `json:"cluster,omitempty"` 39 | 40 | // Tower is the config of tower. 41 | Tower Tower `json:"tower,omitempty"` 42 | 43 | // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. 44 | // +optional 45 | ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` 46 | 47 | // VMGracefulShutdown indicates the VMs in this ElfCluster should shutdown gracefully when deleting the VMs. 48 | // Default to false because sometimes the OS stuck when shutting down gracefully. 49 | // +optional 50 | VMGracefulShutdown bool `json:"vmGracefulShutdown,omitempty"` 51 | } 52 | 53 | // ElfClusterStatus defines the observed state of ElfCluster. 54 | type ElfClusterStatus struct { 55 | // +optional 56 | Ready bool `json:"ready,omitempty"` 57 | 58 | // Conditions defines current service state of the ElfCluster. 59 | // +optional 60 | Conditions clusterv1.Conditions `json:"conditions,omitempty"` 61 | } 62 | 63 | //+kubebuilder:object:root=true 64 | //+kubebuilder:subresource:status 65 | //+kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready" 66 | //+kubebuilder:printcolumn:name="Tower",type="string",JSONPath=".spec.tower.server",description="Tower is the address of the Tower endpoint" 67 | //+kubebuilder:printcolumn:name="ControlPlaneEndpoint",type="string",JSONPath=".spec.controlPlaneEndpoint",description="API Endpoint",priority=1 68 | //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Cluster" 69 | 70 | // ElfCluster is the Schema for the elfclusters API. 71 | type ElfCluster struct { 72 | metav1.TypeMeta `json:",inline"` 73 | metav1.ObjectMeta `json:"metadata,omitempty"` 74 | 75 | Spec ElfClusterSpec `json:"spec,omitempty"` 76 | Status ElfClusterStatus `json:"status,omitempty"` 77 | } 78 | 79 | func (c *ElfCluster) GetConditions() clusterv1.Conditions { 80 | return c.Status.Conditions 81 | } 82 | 83 | func (c *ElfCluster) SetConditions(conditions clusterv1.Conditions) { 84 | c.Status.Conditions = conditions 85 | } 86 | 87 | func (c *ElfCluster) GetTower() Tower { 88 | return c.Spec.Tower 89 | } 90 | 91 | func (c *ElfCluster) HasForceDeleteCluster() bool { 92 | if c.Annotations == nil { 93 | return false 94 | } 95 | _, ok := c.Annotations[ElfClusterForceDeleteAnnotation] 96 | return ok 97 | } 98 | 99 | //+kubebuilder:object:root=true 100 | 101 | // ElfClusterList contains a list of ElfCluster. 102 | type ElfClusterList struct { 103 | metav1.TypeMeta `json:",inline"` 104 | metav1.ListMeta `json:"metadata,omitempty"` 105 | Items []ElfCluster `json:"items"` 106 | } 107 | 108 | func init() { 109 | SchemeBuilder.Register(&ElfCluster{}, &ElfClusterList{}) 110 | } 111 | -------------------------------------------------------------------------------- /api/v1beta1/elfmachinetemplate_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // ElfMachineTemplateSpec defines the desired state of ElfMachineTemplate. 24 | type ElfMachineTemplateSpec struct { 25 | Template ElfMachineTemplateResource `json:"template"` 26 | } 27 | 28 | //+kubebuilder:object:root=true 29 | 30 | // ElfMachineTemplate is the Schema for the elfmachinetemplates API. 31 | type ElfMachineTemplate struct { 32 | metav1.TypeMeta `json:",inline"` 33 | metav1.ObjectMeta `json:"metadata,omitempty"` 34 | 35 | Spec ElfMachineTemplateSpec `json:"spec,omitempty"` 36 | } 37 | 38 | //+kubebuilder:object:root=true 39 | 40 | // ElfMachineTemplateList contains a list of ElfMachineTemplate. 41 | type ElfMachineTemplateList struct { 42 | metav1.TypeMeta `json:",inline"` 43 | metav1.ListMeta `json:"metadata,omitempty"` 44 | Items []ElfMachineTemplate `json:"items"` 45 | } 46 | 47 | func init() { 48 | SchemeBuilder.Register(&ElfMachineTemplate{}, &ElfMachineTemplateList{}) 49 | } 50 | -------------------------------------------------------------------------------- /api/v1beta1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1beta1 contains API Schema definitions for the infrastructure v1beta1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=infrastructure.cluster.x-k8s.io 20 | package v1beta1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects. 29 | GroupVersion = schema.GroupVersion{Group: "infrastructure.cluster.x-k8s.io", Version: "v1beta1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme. 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /clusterctl-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "infrastructure-elf", 3 | "config": { 4 | "componentsFile": "infrastructure-components.yaml", 5 | "nextVersion": "v0.2.1" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. 4 | apiVersion: cert-manager.io/v1 5 | kind: Issuer 6 | metadata: 7 | labels: 8 | app.kubernetes.io/name: issuer 9 | app.kubernetes.io/instance: selfsigned-issuer 10 | app.kubernetes.io/component: certificate 11 | app.kubernetes.io/created-by: cluster-api-provider-elf 12 | app.kubernetes.io/part-of: cluster-api-provider-elf 13 | app.kubernetes.io/managed-by: kustomize 14 | name: selfsigned-issuer 15 | namespace: system 16 | spec: 17 | selfSigned: {} 18 | --- 19 | apiVersion: cert-manager.io/v1 20 | kind: Certificate 21 | metadata: 22 | labels: 23 | app.kubernetes.io/name: certificate 24 | app.kubernetes.io/instance: serving-cert 25 | app.kubernetes.io/component: certificate 26 | app.kubernetes.io/created-by: cluster-api-provider-elf 27 | app.kubernetes.io/part-of: cluster-api-provider-elf 28 | app.kubernetes.io/managed-by: kustomize 29 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 30 | namespace: system 31 | spec: 32 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 33 | dnsNames: 34 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 35 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 36 | issuerRef: 37 | kind: Issuer 38 | name: selfsigned-issuer 39 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 40 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | commonLabels: 2 | cluster.x-k8s.io/v1beta1: v1beta1 3 | 4 | apiVersion: kustomize.config.k8s.io/v1beta1 5 | kind: Kustomization 6 | # This kustomization.yaml is not intended to be run by itself, 7 | # since it depends on service name and namespace that are out of this kustomize package. 8 | # It should be run by config/default 9 | resources: 10 | - bases/infrastructure.cluster.x-k8s.io_elfclusters.yaml 11 | - bases/infrastructure.cluster.x-k8s.io_elfmachines.yaml 12 | - bases/infrastructure.cluster.x-k8s.io_elfmachinetemplates.yaml 13 | #+kubebuilder:scaffold:crdkustomizeresource 14 | 15 | patchesStrategicMerge: 16 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 17 | # patches here are for enabling the conversion webhook for each CRD 18 | #- patches/webhook_in_elfclusters.yaml 19 | #- patches/webhook_in_elfmachines.yaml 20 | #- patches/webhook_in_elfmachinetemplates.yaml 21 | #+kubebuilder:scaffold:crdkustomizewebhookpatch 22 | 23 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 24 | # patches here are for enabling the CA injection for each CRD 25 | #- patches/cainjection_in_elfclusters.yaml 26 | #- patches/cainjection_in_elfmachines.yaml 27 | #- patches/cainjection_in_elfmachinetemplates.yaml 28 | #+kubebuilder:scaffold:crdkustomizecainjectionpatch 29 | 30 | # the following config is for teaching kustomize how to do kustomization for CRDs. 31 | configurations: 32 | - kustomizeconfig.yaml 33 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_elfclusters.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: elfclusters.infrastructure.cluster.x-k8s.io 8 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_elfmachines.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: elfmachines.infrastructure.cluster.x-k8s.io 8 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_elfmachinetemplates.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: elfmachinetemplates.infrastructure.cluster.x-k8s.io 8 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_elfclusters.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: elfclusters.infrastructure.cluster.x-k8s.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_elfmachines.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: elfmachines.infrastructure.cluster.x-k8s.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_elfmachinetemplates.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: elfmachinetemplates.infrastructure.cluster.x-k8s.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | # Adds namespace to all resources. 5 | namespace: cape-system 6 | 7 | namePrefix: cape- 8 | 9 | commonLabels: 10 | cluster.x-k8s.io/provider: "infrastructure-elf" 11 | 12 | resources: 13 | - namespace.yaml 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | 20 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 21 | # crd/kustomization.yaml 22 | - ../webhook 23 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 24 | - ../certmanager 25 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 26 | #- ../prometheus 27 | 28 | patchesStrategicMerge: 29 | - manager_image_patch.yaml 30 | - manager_pull_policy.yaml 31 | # Protect the /metrics endpoint by putting it behind auth. 32 | # If you want your controller-manager to expose the /metrics 33 | # endpoint w/o any authn/z, please comment the following line. 34 | 35 | # Mount the controller config file for loading manager configurations 36 | # through a ComponentConfig type 37 | #- manager_config_patch.yaml 38 | 39 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 40 | # crd/kustomization.yaml 41 | - manager_webhook_patch.yaml 42 | 43 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 44 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 45 | # 'CERTMANAGER' needs to be enabled to use ca injection 46 | - webhookcainjection_patch.yaml 47 | 48 | # the following config is for teaching kustomize how to do var substitution 49 | vars: 50 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 51 | - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 52 | objref: 53 | kind: Certificate 54 | group: cert-manager.io 55 | version: v1 56 | name: serving-cert # this name should match the one in certificate.yaml 57 | fieldref: 58 | fieldpath: metadata.namespace 59 | - name: CERTIFICATE_NAME 60 | objref: 61 | kind: Certificate 62 | group: cert-manager.io 63 | version: v1 64 | name: serving-cert # this name should match the one in certificate.yaml 65 | - name: SERVICE_NAMESPACE # namespace of the service 66 | objref: 67 | kind: Service 68 | version: v1 69 | name: webhook-service 70 | fieldref: 71 | fieldpath: metadata.namespace 72 | - name: SERVICE_NAME 73 | objref: 74 | kind: Service 75 | version: v1 76 | name: webhook-service 77 | -------------------------------------------------------------------------------- /config/default/manager_config_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | args: 12 | - "--config=controller_manager_config.yaml" 13 | volumeMounts: 14 | - name: manager-config 15 | mountPath: /controller_manager_config.yaml 16 | subPath: controller_manager_config.yaml 17 | volumes: 18 | - name: manager-config 19 | configMap: 20 | name: manager-config 21 | -------------------------------------------------------------------------------- /config/default/manager_image_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - image: docker.io/smartxworks/cape-manager:latest 11 | name: manager 12 | -------------------------------------------------------------------------------- /config/default/manager_pull_policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | imagePullPolicy: IfNotPresent 12 | -------------------------------------------------------------------------------- /config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | ports: 12 | - containerPort: 9443 13 | name: webhook-server 14 | protocol: TCP 15 | volumeMounts: 16 | - mountPath: /tmp/k8s-webhook-server/serving-certs 17 | name: cert 18 | readOnly: true 19 | volumes: 20 | - name: cert 21 | secret: 22 | defaultMode: 420 23 | secretName: webhook-server-cert 24 | -------------------------------------------------------------------------------- /config/default/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: system 5 | -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | apiVersion: admissionregistration.k8s.io/v1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | labels: 7 | app.kubernetes.io/name: mutatingwebhookconfiguration 8 | app.kubernetes.io/instance: mutating-webhook-configuration 9 | app.kubernetes.io/component: webhook 10 | app.kubernetes.io/created-by: cluster-api-provider-elf 11 | app.kubernetes.io/part-of: cluster-api-provider-elf 12 | app.kubernetes.io/managed-by: kustomize 13 | name: mutating-webhook-configuration 14 | annotations: 15 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 16 | --- 17 | apiVersion: admissionregistration.k8s.io/v1 18 | kind: ValidatingWebhookConfiguration 19 | metadata: 20 | labels: 21 | app.kubernetes.io/name: validatingwebhookconfiguration 22 | app.kubernetes.io/instance: validating-webhook-configuration 23 | app.kubernetes.io/component: webhook 24 | app.kubernetes.io/created-by: cluster-api-provider-elf 25 | app.kubernetes.io/part-of: cluster-api-provider-elf 26 | app.kubernetes.io/managed-by: kustomize 27 | name: validating-webhook-configuration 28 | annotations: 29 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 30 | -------------------------------------------------------------------------------- /config/manager/controller_manager_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 2 | kind: ControllerManagerConfig 3 | health: 4 | healthProbeBindAddress: :8081 5 | metrics: 6 | bindAddress: 127.0.0.1:8080 7 | webhook: 8 | port: 9443 9 | leaderElection: 10 | leaderElect: true 11 | resourceName: ea6b6891.cluster.x-k8s.io 12 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - manager.yaml 5 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | labels: 7 | control-plane: controller-manager 8 | spec: 9 | selector: 10 | matchLabels: 11 | control-plane: controller-manager 12 | replicas: 2 13 | template: 14 | metadata: 15 | labels: 16 | control-plane: controller-manager 17 | spec: 18 | affinity: 19 | podAntiAffinity: 20 | preferredDuringSchedulingIgnoredDuringExecution: 21 | - podAffinityTerm: 22 | labelSelector: 23 | matchLabels: 24 | cluster.x-k8s.io/provider: infrastructure-elf 25 | control-plane: controller-manager 26 | topologyKey: kubernetes.io/hostname 27 | namespaces: 28 | - cape-system 29 | weight: 1 30 | containers: 31 | - args: 32 | - --leader-elect 33 | - --v=4 34 | - --max-concurrent-vm-creations=20 35 | image: docker.io/smartxworks/cape-manager:latest 36 | imagePullPolicy: IfNotPresent 37 | name: manager 38 | env: 39 | - name: TOWER_RESOURCE_PREFIX 40 | value: cape 41 | - name: POD_NAMESPACE 42 | valueFrom: 43 | fieldRef: 44 | fieldPath: metadata.namespace 45 | - name: POD_NAME 46 | valueFrom: 47 | fieldRef: 48 | fieldPath: metadata.name 49 | - name: POD_UID 50 | valueFrom: 51 | fieldRef: 52 | fieldPath: metadata.uid 53 | ports: 54 | - containerPort: 9440 55 | name: healthz 56 | protocol: TCP 57 | readinessProbe: 58 | httpGet: 59 | path: /readyz 60 | port: healthz 61 | livenessProbe: 62 | httpGet: 63 | path: /healthz 64 | port: healthz 65 | terminationMessagePolicy: FallbackToLogsOnError 66 | serviceAccountName: controller-manager 67 | terminationGracePeriodSeconds: 10 68 | tolerations: 69 | - effect: NoSchedule 70 | key: node-role.kubernetes.io/master 71 | - effect: NoSchedule 72 | key: node-role.kubernetes.io/control-plane 73 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | scheme: https 15 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 16 | tlsConfig: 17 | insecureSkipVerify: true 18 | selector: 19 | matchLabels: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /config/rbac/elfcluster_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit elfclusters. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: elfcluster-editor-role 6 | rules: 7 | - apiGroups: 8 | - infrastructure.cluster.x-k8s.io 9 | resources: 10 | - elfclusters 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - infrastructure.cluster.x-k8s.io 21 | resources: 22 | - elfclusters/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/elfcluster_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view elfclusters. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: elfcluster-viewer-role 6 | rules: 7 | - apiGroups: 8 | - infrastructure.cluster.x-k8s.io 9 | resources: 10 | - elfclusters 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - infrastructure.cluster.x-k8s.io 17 | resources: 18 | - elfclusters/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/elfmachine_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit elfmachines. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: elfmachine-editor-role 6 | rules: 7 | - apiGroups: 8 | - infrastructure.cluster.x-k8s.io 9 | resources: 10 | - elfmachines 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - infrastructure.cluster.x-k8s.io 21 | resources: 22 | - elfmachines/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/elfmachine_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view elfmachines. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: elfmachine-viewer-role 6 | rules: 7 | - apiGroups: 8 | - infrastructure.cluster.x-k8s.io 9 | resources: 10 | - elfmachines 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - infrastructure.cluster.x-k8s.io 17 | resources: 18 | - elfmachines/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/elfmachinetemplate_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit elfmachinetemplates. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: elfmachinetemplate-editor-role 6 | rules: 7 | - apiGroups: 8 | - infrastructure.cluster.x-k8s.io 9 | resources: 10 | - elfmachinetemplates 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - infrastructure.cluster.x-k8s.io 21 | resources: 22 | - elfmachinetemplates/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/elfmachinetemplate_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view elfmachinetemplates. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: elfmachinetemplate-viewer-role 6 | rules: 7 | - apiGroups: 8 | - infrastructure.cluster.x-k8s.io 9 | resources: 10 | - elfmachinetemplates 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - infrastructure.cluster.x-k8s.io 17 | resources: 18 | - elfmachinetemplates/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | # All RBAC will be applied under this service account in 5 | # the deployment namespace. You may comment out this resource 6 | # if your manager will use a service account that exists at 7 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 8 | # subjects if changing service account names. 9 | - role.yaml 10 | - role_binding.yaml 11 | - service_account.yaml 12 | - leader_election_role.yaml 13 | - leader_election_role_binding.yaml 14 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - coordination.k8s.io 21 | resources: 22 | - leases 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - events 35 | verbs: 36 | - create 37 | - patch 38 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: manager-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - events 11 | verbs: 12 | - create 13 | - get 14 | - list 15 | - patch 16 | - update 17 | - watch 18 | - apiGroups: 19 | - cluster.x-k8s.io 20 | resources: 21 | - clusters 22 | - clusters/status 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - apiGroups: 28 | - cluster.x-k8s.io 29 | resources: 30 | - machinedeployments 31 | verbs: 32 | - create 33 | - delete 34 | - get 35 | - list 36 | - patch 37 | - update 38 | - watch 39 | - apiGroups: 40 | - cluster.x-k8s.io 41 | resources: 42 | - machinedeployments 43 | - machinedeployments/status 44 | verbs: 45 | - get 46 | - list 47 | - watch 48 | - apiGroups: 49 | - cluster.x-k8s.io 50 | resources: 51 | - machines 52 | - machines/status 53 | verbs: 54 | - get 55 | - list 56 | - patch 57 | - watch 58 | - apiGroups: 59 | - controlplane.cluster.x-k8s.io 60 | resources: 61 | - '*' 62 | verbs: 63 | - create 64 | - delete 65 | - get 66 | - list 67 | - patch 68 | - update 69 | - watch 70 | - apiGroups: 71 | - "" 72 | resources: 73 | - secrets 74 | verbs: 75 | - create 76 | - get 77 | - list 78 | - patch 79 | - watch 80 | - apiGroups: 81 | - infrastructure.cluster.x-k8s.io 82 | resources: 83 | - elfclusters 84 | verbs: 85 | - create 86 | - delete 87 | - get 88 | - list 89 | - patch 90 | - update 91 | - watch 92 | - apiGroups: 93 | - infrastructure.cluster.x-k8s.io 94 | resources: 95 | - elfclusters/finalizers 96 | verbs: 97 | - update 98 | - apiGroups: 99 | - infrastructure.cluster.x-k8s.io 100 | resources: 101 | - elfclusters/status 102 | verbs: 103 | - get 104 | - patch 105 | - update 106 | - apiGroups: 107 | - infrastructure.cluster.x-k8s.io 108 | resources: 109 | - elfmachines 110 | verbs: 111 | - create 112 | - delete 113 | - get 114 | - list 115 | - patch 116 | - update 117 | - watch 118 | - apiGroups: 119 | - infrastructure.cluster.x-k8s.io 120 | resources: 121 | - elfmachines/finalizers 122 | verbs: 123 | - update 124 | - apiGroups: 125 | - infrastructure.cluster.x-k8s.io 126 | resources: 127 | - elfmachines/status 128 | verbs: 129 | - get 130 | - patch 131 | - update 132 | - apiGroups: 133 | - infrastructure.cluster.x-k8s.io 134 | resources: 135 | - elfmachinetemplates 136 | verbs: 137 | - get 138 | - list 139 | - patch 140 | - update 141 | - watch 142 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | -------------------------------------------------------------------------------- /config/samples/infrastructure_v1alpha3_elfcluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 2 | kind: ElfCluster 3 | metadata: 4 | name: elfcluster-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /config/samples/infrastructure_v1alpha3_elfmachine.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 2 | kind: ElfMachine 3 | metadata: 4 | name: elfmachine-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /config/samples/infrastructure_v1alpha3_elfmachinetemplate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 2 | kind: ElfMachineTemplate 3 | metadata: 4 | name: elfmachinetemplate-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /config/webhook/manifests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: admissionregistration.k8s.io/v1 3 | kind: MutatingWebhookConfiguration 4 | metadata: 5 | name: mutating-webhook-configuration 6 | webhooks: 7 | - admissionReviewVersions: 8 | - v1 9 | clientConfig: 10 | service: 11 | name: webhook-service 12 | namespace: system 13 | path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine 14 | failurePolicy: Fail 15 | name: mutation.elfmachine.infrastructure.x-k8s.io 16 | rules: 17 | - apiGroups: 18 | - infrastructure.cluster.x-k8s.io 19 | apiVersions: 20 | - v1beta1 21 | operations: 22 | - CREATE 23 | - UPDATE 24 | resources: 25 | - elfmachines 26 | sideEffects: None 27 | - admissionReviewVersions: 28 | - v1 29 | clientConfig: 30 | service: 31 | name: webhook-service 32 | namespace: system 33 | path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachinetemplate 34 | failurePolicy: Fail 35 | name: mutation.elfmachinetemplate.infrastructure.x-k8s.io 36 | rules: 37 | - apiGroups: 38 | - infrastructure.cluster.x-k8s.io 39 | apiVersions: 40 | - v1beta1 41 | operations: 42 | - CREATE 43 | - UPDATE 44 | resources: 45 | - elfmachinetemplates 46 | sideEffects: None 47 | --- 48 | apiVersion: admissionregistration.k8s.io/v1 49 | kind: ValidatingWebhookConfiguration 50 | metadata: 51 | name: validating-webhook-configuration 52 | webhooks: 53 | - admissionReviewVersions: 54 | - v1 55 | clientConfig: 56 | service: 57 | name: webhook-service 58 | namespace: system 59 | path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine 60 | failurePolicy: Fail 61 | name: validation.elfmachine.infrastructure.x-k8s.io 62 | rules: 63 | - apiGroups: 64 | - infrastructure.cluster.x-k8s.io 65 | apiVersions: 66 | - v1beta1 67 | operations: 68 | - CREATE 69 | - UPDATE 70 | resources: 71 | - elfmachines 72 | sideEffects: None 73 | - admissionReviewVersions: 74 | - v1 75 | clientConfig: 76 | service: 77 | name: webhook-service 78 | namespace: system 79 | path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachinetemplate 80 | failurePolicy: Fail 81 | name: validation.elfmachinetemplate.infrastructure.x-k8s.io 82 | rules: 83 | - apiGroups: 84 | - infrastructure.cluster.x-k8s.io 85 | apiVersions: 86 | - v1beta1 87 | operations: 88 | - CREATE 89 | - UPDATE 90 | resources: 91 | - elfmachinetemplates 92 | sideEffects: None 93 | -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | labels: 6 | app.kubernetes.io/name: service 7 | app.kubernetes.io/instance: webhook-service 8 | app.kubernetes.io/component: webhook 9 | app.kubernetes.io/created-by: cluster-api-provider-elf 10 | app.kubernetes.io/part-of: cluster-api-provider-elf 11 | app.kubernetes.io/managed-by: kustomize 12 | name: webhook-service 13 | namespace: system 14 | spec: 15 | ports: 16 | - port: 443 17 | protocol: TCP 18 | targetPort: 9443 19 | selector: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /docs/releasing.md: -------------------------------------------------------------------------------- 1 | # 发布流程 2 | 3 | 1. 如果发布的是新的 minor 版本，创建一个新的发布分支并推送到 GitHub，否则切换到该分支，例如 `release-0.6` 4 | 2. 设置版本环境变量 `VERSION=v0.x.x`，版本以 v 为前缀，例如 v0.0.6 5 | 3. 打标签 `git tag -m $VERSION $VERSION` 6 | 4. 推送标签到 GitHub `git push upstream $VERSION` 7 | 5. 标签推送到 GitHub 后会自动触发 [Github Action](https://github.com/smartxworks/cluster-api-provider-elf) 创建一个[待发布版本](https://github.com/smartxworks/cluster-api-provider-elf/releases) 8 | + 5.1 构建 docker 镜像 9 | + 5.2 推送 docker 镜像到镜像仓库 10 | + 5.3 生成 manifests 11 | + 5.4 创建待发布版本 12 | + 5.5 上传 manifests 到待发布版本的 Assets 13 | 6. 检查 GitHub 的待发布版本和镜像 14 | 7. 发布新版本 15 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /hack/fetch_ext_bins.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | # Enable tracing in this script off by setting the TRACE variable in your 22 | # environment to any value: 23 | # 24 | # $ TRACE=1 test.sh 25 | TRACE=${TRACE:-""} 26 | if [[ -n "${TRACE}" ]]; then 27 | set -x 28 | fi 29 | 30 | k8s_version=1.30.0 31 | goarch=amd64 32 | goos="unknown" 33 | 34 | if [[ "${OSTYPE}" == "linux"* ]]; then 35 | goos="linux" 36 | elif [[ "${OSTYPE}" == "darwin"* ]]; then 37 | goos="darwin" 38 | fi 39 | 40 | if [[ "$goos" == "unknown" ]]; then 41 | echo "OS '$OSTYPE' not supported. Aborting." >&2 42 | exit 1 43 | fi 44 | 45 | # Turn colors in this script off by setting the NO_COLOR variable in your 46 | # environment to any value: 47 | # 48 | # $ NO_COLOR=1 test.sh 49 | NO_COLOR=${NO_COLOR:-""} 50 | if [[ -z "${NO_COLOR}" ]]; then 51 | header=$'\e[1;33m' 52 | reset=$'\e[0m' 53 | else 54 | header='' 55 | reset='' 56 | fi 57 | 58 | function header_text { 59 | echo "$header$*$reset" 60 | } 61 | 62 | tmp_root=/tmp 63 | 64 | # Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable 65 | # in your environment to any value: 66 | # 67 | # $ SKIP_FETCH_TOOLS=1 ./fetch_ext_bins.sh 68 | # 69 | # If you skip fetching tools, this script will use the tools already on your 70 | # machine. 71 | SKIP_FETCH_TOOLS=${SKIP_FETCH_TOOLS:-""} 72 | 73 | function fetch_tools { 74 | if [[ -n "$SKIP_FETCH_TOOLS" ]]; then 75 | return 0 76 | fi 77 | 78 | mkdir -p ${tmp_root} 79 | 80 | # use the pre-existing version in the temporary folder if it matches our k8s version 81 | if [[ -x "${tmp_root}/kubebuilder/bin/kube-apiserver" ]]; then 82 | version=$(${tmp_root}/kubebuilder/bin/kube-apiserver --version) 83 | if [[ $version == *"${k8s_version}"* ]]; then 84 | return 0 85 | fi 86 | fi 87 | 88 | header_text "fetching kubebuilder-tools@${k8s_version}" 89 | kb_tools_archive_name="kubebuilder-tools-${k8s_version}-${goos}-${goarch}.tar.gz" 90 | kb_tools_download_url="https://storage.googleapis.com/kubebuilder-tools/${kb_tools_archive_name}" 91 | 92 | kb_tools_archive_path="${tmp_root}/${kb_tools_archive_name}" 93 | if [[ ! -f ${kb_tools_archive_path} ]]; then 94 | curl -fsL ${kb_tools_download_url} -o "${kb_tools_archive_path}" 95 | fi 96 | tar -zvxf "${kb_tools_archive_path}" -C "${tmp_root}/" 97 | rm "${kb_tools_archive_path}" 98 | } 99 | 100 | function setup_envs { 101 | header_text "setting up kubebuilder-tools@${k8s_version} env vars" 102 | 103 | # Setup env vars 104 | export PATH=${tmp_root}/kubebuilder/bin:$PATH 105 | export TEST_ASSET_KUBECTL=${tmp_root}/kubebuilder/bin/kubectl 106 | export TEST_ASSET_KUBE_APISERVER=${tmp_root}/kubebuilder/bin/kube-apiserver 107 | export TEST_ASSET_ETCD=${tmp_root}/kubebuilder/bin/etcd 108 | } 109 | -------------------------------------------------------------------------------- /hack/version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2020 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | version::get_version_vars() { 21 | # shellcheck disable=SC1083 22 | GIT_COMMIT="$(git rev-parse HEAD^{commit})" 23 | 24 | if git_status=$(git status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then 25 | GIT_TREE_STATE="clean" 26 | else 27 | GIT_TREE_STATE="dirty" 28 | fi 29 | 30 | # stolen from k8s.io/hack/lib/version.sh 31 | # Use git describe to find the version based on tags. 32 | if GIT_VERSION=$(git describe --tags --abbrev=14 2>/dev/null); then 33 | # This translates the "git describe" to an actual semver.org 34 | # compatible semantic version that looks something like this: 35 | # v1.1.0-alpha.0.6+84c76d1142ea4d 36 | # 37 | # TODO: We continue calling this "git version" because so many 38 | # downstream consumers are expecting it there. 39 | # shellcheck disable=SC2001 40 | DASHES_IN_VERSION=$(echo "${GIT_VERSION}" | sed "s/[^-]//g") 41 | if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then 42 | # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) 43 | # shellcheck disable=SC2001 44 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\-\2/") 45 | elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then 46 | # We have distance to base tag (v1.1.0-1-gCommitHash) 47 | # shellcheck disable=SC2001 48 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/-\1/") 49 | fi 50 | if [[ "${GIT_TREE_STATE}" == "dirty" ]]; then 51 | # git describe --dirty only considers changes to existing files, but 52 | # that is problematic since new untracked .go files affect the build, 53 | # so use our idea of "dirty" from git status instead. 54 | GIT_VERSION+="-dirty" 55 | fi 56 | 57 | 58 | # Try to match the "git describe" output to a regex to try to extract 59 | # the "major" and "minor" versions and whether this is the exact tagged 60 | # version or whether the tree is between two tagged versions. 61 | if [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then 62 | GIT_MAJOR=${BASH_REMATCH[1]} 63 | GIT_MINOR=${BASH_REMATCH[2]} 64 | fi 65 | 66 | # If GIT_VERSION is not a valid Semantic Version, then refuse to build. 67 | if ! [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then 68 | echo "GIT_VERSION should be a valid Semantic Version. Current value: ${GIT_VERSION}" 69 | echo "Please see more details here: https://semver.org" 70 | exit 1 71 | fi 72 | fi 73 | 74 | GIT_RELEASE_TAG=$(git describe --abbrev=0 --tags) 75 | GIT_RELEASE_COMMIT=$(git rev-list -n 1 "${GIT_RELEASE_TAG}") 76 | } 77 | 78 | # stolen from k8s.io/hack/lib/version.sh and modified 79 | # Prints the value that needs to be passed to the -ldflags parameter of go build 80 | version::ldflags() { 81 | version::get_version_vars 82 | 83 | local -a ldflags 84 | function add_ldflag() { 85 | local key=${1} 86 | local val=${2} 87 | ldflags+=( 88 | "-X 'github.com/smartxworks/cluster-api-provider-elf/pkg/version.${key}=${val}'" 89 | ) 90 | } 91 | 92 | add_ldflag "buildVersion" $1 93 | add_ldflag "buildDate" "$(date ${SOURCE_DATE_EPOCH:+"--date=@${SOURCE_DATE_EPOCH}"} -u +'%Y-%m-%dT%H:%M:%SZ')" 94 | add_ldflag "gitCommit" "${GIT_COMMIT}" 95 | add_ldflag "gitTreeState" "${GIT_TREE_STATE}" 96 | add_ldflag "gitMajor" "${GIT_MAJOR}" 97 | add_ldflag "gitMinor" "${GIT_MINOR}" 98 | add_ldflag "gitVersion" "${GIT_VERSION}" 99 | add_ldflag "gitReleaseCommit" "${GIT_RELEASE_COMMIT}" 100 | 101 | # The -ldflags parameter takes a single string, so join the output. 102 | echo "${ldflags[*]-}" 103 | } 104 | 105 | version::ldflags $1 106 | -------------------------------------------------------------------------------- /metadata.yaml: -------------------------------------------------------------------------------- 1 | # maps release series of major.minor to cluster-api contract version 2 | # the contract version may change between minor or major versions, but *not* 3 | # between patch versions. 4 | # 5 | # update this file only when a new major or minor version is released 6 | apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 7 | kind: Metadata 8 | releaseSeries: 9 | - major: 1 10 | minor: 6 11 | contract: v1beta1 12 | - major: 1 13 | minor: 5 14 | contract: v1beta1 15 | - major: 1 16 | minor: 4 17 | contract: v1beta1 18 | - major: 1 19 | minor: 3 20 | contract: v1beta1 21 | - major: 1 22 | minor: 2 23 | contract: v1beta1 24 | - major: 1 25 | minor: 1 26 | contract: v1beta1 27 | - major: 1 28 | minor: 0 29 | contract: v1beta1 30 | - major: 0 31 | minor: 4 32 | contract: v1beta1 33 | - major: 0 34 | minor: 3 35 | contract: v1beta1 36 | - major: 0 37 | minor: 2 38 | contract: v1beta1 39 | - major: 0 40 | minor: 1 41 | contract: v1beta1 42 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import "time" 20 | 21 | var ( 22 | ProviderNameShort = "cape" 23 | 24 | // DefaultRequeueTimeout is the default time for how long to wait when 25 | // requeueing a CAPE operation. 26 | DefaultRequeueTimeout = 10 * time.Second 27 | 28 | // WaitTaskInterval is the default interval time polling task. 29 | WaitTaskInterval = 1 * time.Second 30 | 31 | // WaitTaskTimeout is the default timeout for waiting for task to complete. 32 | WaitTaskTimeout = 3 * time.Second 33 | 34 | // WaitTaskTimeoutForPlacementGroupOperation is the timeout for waiting for placement group creating/updating/deleting task to complete. 35 | WaitTaskTimeoutForPlacementGroupOperation = 10 * time.Second 36 | 37 | // VMPowerStatusCheckingDuration is the time duration for cheking if the VM is powered off 38 | // after the Machine's NodeHealthy condition status is set to Unknown. 39 | VMPowerStatusCheckingDuration = 2 * time.Minute 40 | ) 41 | -------------------------------------------------------------------------------- /pkg/config/vm.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | const ( 20 | // VMDescription is the default description in a VM. 21 | VMDescription = "Automatically created Kubernetes node by server %s." 22 | 23 | // VMNumCPUs is the default virtual processors in a VM. 24 | VMNumCPUs = 2 25 | 26 | // VMMemoryMiB is the default memory in a VM. 27 | VMMemoryMiB = 2048 28 | 29 | // VMDiskName is the default disk name in a VM. 30 | VMDiskName = "disk" 31 | ) 32 | 33 | // MaxConcurrentVMCreations is the maximum number of concurrent virtual machine creations. 34 | var MaxConcurrentVMCreations = 20 35 | -------------------------------------------------------------------------------- /pkg/constants/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package constants 18 | -------------------------------------------------------------------------------- /pkg/context/cluster_context.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package context 18 | 19 | import ( 20 | goctx "context" 21 | "fmt" 22 | 23 | "github.com/go-logr/logr" 24 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 25 | "sigs.k8s.io/cluster-api/util/patch" 26 | 27 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 28 | "github.com/smartxworks/cluster-api-provider-elf/pkg/service" 29 | ) 30 | 31 | // ClusterContext is a Go context used with a ElfCluster. 32 | type ClusterContext struct { 33 | Cluster *clusterv1.Cluster 34 | ElfCluster *infrav1.ElfCluster 35 | PatchHelper *patch.Helper 36 | Logger logr.Logger 37 | VMService service.VMService 38 | } 39 | 40 | // String returns ElfClusterGroupVersionKind ElfClusterNamespace/ElfClusterName. 41 | func (r *ClusterContext) String() string { 42 | return fmt.Sprintf("%s %s/%s", r.ElfCluster.GroupVersionKind(), r.ElfCluster.Namespace, r.ElfCluster.Name) 43 | } 44 | 45 | // Patch updates the object and its status on the API server. 46 | func (r *ClusterContext) Patch(ctx goctx.Context) error { 47 | return r.PatchHelper.Patch(ctx, r.ElfCluster) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/context/controller_manager_context.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package context 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | "sigs.k8s.io/controller-runtime/pkg/cache" 22 | "sigs.k8s.io/controller-runtime/pkg/client" 23 | ) 24 | 25 | // ControllerManagerContext is the context of the controller that owns the 26 | // controllers. 27 | type ControllerManagerContext struct { 28 | // Namespace is the namespace in which the resource is located responsible 29 | // for running the controller manager. 30 | Namespace string 31 | 32 | // Name is the name of the controller manager. 33 | Name string 34 | 35 | // LeaderElectionID is the information used to identify the object 36 | // responsible for synchronizing leader election. 37 | LeaderElectionID string 38 | 39 | // LeaderElectionNamespace is the namespace in which the LeaderElection 40 | // object is located. 41 | LeaderElectionNamespace string 42 | 43 | // WatchNamespaces are the namespaces the controllers watches for changes. If 44 | // no value is specified then all namespaces are watched. 45 | WatchNamespaces map[string]cache.Config 46 | 47 | // Client is the controller manager's client. 48 | Client client.Client 49 | 50 | // Scheme is the controller manager's API scheme. 51 | Scheme *runtime.Scheme 52 | 53 | // WatchFilterValue is used to filter incoming objects by label. 54 | WatchFilterValue string 55 | } 56 | 57 | // String returns ControllerManagerName. 58 | func (r *ControllerManagerContext) String() string { 59 | return r.Name 60 | } 61 | -------------------------------------------------------------------------------- /pkg/context/machine_context.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package context 18 | 19 | import ( 20 | goctx "context" 21 | "fmt" 22 | 23 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 24 | "sigs.k8s.io/cluster-api/util/patch" 25 | 26 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 27 | "github.com/smartxworks/cluster-api-provider-elf/pkg/service" 28 | ) 29 | 30 | // MachineContext is a Go context used with an ElfMachine. 31 | type MachineContext struct { 32 | Cluster *clusterv1.Cluster 33 | Machine *clusterv1.Machine 34 | ElfCluster *infrav1.ElfCluster 35 | ElfMachine *infrav1.ElfMachine 36 | PatchHelper *patch.Helper 37 | VMService service.VMService 38 | } 39 | 40 | // String returns ElfMachineGroupVersionKindElfMachineNamespace/ElfMachineName. 41 | func (c *MachineContext) String() string { 42 | return fmt.Sprintf("%s %s/%s", c.ElfMachine.GroupVersionKind(), c.ElfMachine.Namespace, c.ElfMachine.Name) 43 | } 44 | 45 | // Patch updates the object and its status on the API server. 46 | func (c *MachineContext) Patch(ctx goctx.Context) error { 47 | return c.PatchHelper.Patch(ctx, c.ElfMachine) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/context/machine_template_context.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package context 18 | 19 | import ( 20 | "fmt" 21 | 22 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 23 | 24 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 25 | "github.com/smartxworks/cluster-api-provider-elf/pkg/service" 26 | ) 27 | 28 | // MachineTemplateContext is a Go context used with an ElfMachineTemplate. 29 | type MachineTemplateContext struct { 30 | Cluster *clusterv1.Cluster 31 | ElfCluster *infrav1.ElfCluster 32 | ElfMachineTemplate *infrav1.ElfMachineTemplate 33 | VMService service.VMService 34 | } 35 | 36 | // String returns ElfMachineTemplateGroupVersionKindElfMachineTemplateNamespace/ElfMachineTemplateName. 37 | func (c *MachineTemplateContext) String() string { 38 | return fmt.Sprintf("%s %s/%s", c.ElfMachineTemplate.GroupVersionKind(), c.ElfMachineTemplate.Namespace, c.ElfMachineTemplate.Name) 39 | } 40 | -------------------------------------------------------------------------------- /pkg/errors/consts.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package errors 18 | 19 | import ( 20 | capierrors "sigs.k8s.io/cluster-api/errors" 21 | ) 22 | 23 | const ( 24 | // CloudInitConfigError indicates an error occurred while configuring cloud-init. 25 | CloudInitConfigError capierrors.MachineStatusError = "CloudInitConfigError" 26 | 27 | // RemovedFromInfrastructureError indicates an error that the VM could not be found from the infrastructure. 28 | RemovedFromInfrastructureError capierrors.MachineStatusError = "RemovedFromInfrastructure" 29 | 30 | // MovedToRecycleBinError indicates an error that the VM was moved to the recycle bin. 31 | MovedToRecycleBinError capierrors.MachineStatusError = "MovedToRecycleBin" 32 | ) 33 | -------------------------------------------------------------------------------- /pkg/hostagent/service.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package hostagent 15 | 16 | import ( 17 | goctx "context" 18 | "fmt" 19 | "time" 20 | 21 | agentv1 "github.com/smartxworks/host-config-agent-api/api/v1alpha1" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | apitypes "k8s.io/apimachinery/pkg/types" 24 | "sigs.k8s.io/controller-runtime/pkg/client" 25 | 26 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 27 | "github.com/smartxworks/cluster-api-provider-elf/pkg/hostagent/tasks" 28 | ) 29 | 30 | type HostAgentJobType string 31 | 32 | const ( 33 | defaultTimeout = 1 * time.Minute 34 | 35 | // HostAgentJobTypeExpandRootPartition is the job type for expanding the root partition. 36 | HostAgentJobTypeExpandRootPartition HostAgentJobType = "expand-root-partition" 37 | // HostAgentJobTypeRestartKubelet is the job type for restarting the kubelet. 38 | HostAgentJobTypeRestartKubelet HostAgentJobType = "restart-kubelet" 39 | ) 40 | 41 | func GetHostJob(ctx goctx.Context, c client.Client, namespace, name string) (*agentv1.HostOperationJob, error) { 42 | var restartKubeletJob agentv1.HostOperationJob 43 | if err := c.Get(ctx, apitypes.NamespacedName{ 44 | Name: name, 45 | Namespace: "default", 46 | }, &restartKubeletJob); err != nil { 47 | return nil, err 48 | } 49 | 50 | return &restartKubeletJob, nil 51 | } 52 | 53 | // GetExpandRootPartitionJobName return the expand root partition job name. 54 | // The same disk expansion uses the same job name to reduce duplicate jobs. 55 | func GetExpandRootPartitionJobName(elfMachine *infrav1.ElfMachine) string { 56 | return fmt.Sprintf("cape-expand-root-partition-%s-%d", elfMachine.Name, elfMachine.Spec.DiskGiB) 57 | } 58 | 59 | func GetRestartKubeletJobName(elfMachine *infrav1.ElfMachine) string { 60 | return fmt.Sprintf("cape-restart-kubelet-%s-%d-%d-%d", elfMachine.Name, elfMachine.Spec.NumCPUs, elfMachine.Spec.NumCoresPerSocket, elfMachine.Spec.MemoryMiB) 61 | } 62 | 63 | func GetJobName(elfMachine *infrav1.ElfMachine, jobType HostAgentJobType) string { 64 | switch jobType { 65 | case HostAgentJobTypeExpandRootPartition: 66 | return GetExpandRootPartitionJobName(elfMachine) 67 | case HostAgentJobTypeRestartKubelet: 68 | return GetRestartKubeletJobName(elfMachine) 69 | default: 70 | return "" 71 | } 72 | } 73 | 74 | func GenerateExpandRootPartitionJob(elfMachine *infrav1.ElfMachine) *agentv1.HostOperationJob { 75 | return &agentv1.HostOperationJob{ 76 | ObjectMeta: metav1.ObjectMeta{ 77 | Name: GetExpandRootPartitionJobName(elfMachine), 78 | Namespace: "default", 79 | }, 80 | Spec: agentv1.HostOperationJobSpec{ 81 | NodeName: elfMachine.Name, 82 | Operation: agentv1.Operation{ 83 | Ansible: &agentv1.Ansible{ 84 | LocalPlaybookText: &agentv1.YAMLText{ 85 | Inline: tasks.ExpandRootPartitionTask, 86 | }, 87 | }, 88 | Timeout: metav1.Duration{Duration: defaultTimeout}, 89 | }, 90 | }, 91 | } 92 | } 93 | 94 | func GenerateRestartKubeletJob(elfMachine *infrav1.ElfMachine) *agentv1.HostOperationJob { 95 | return &agentv1.HostOperationJob{ 96 | ObjectMeta: metav1.ObjectMeta{ 97 | Name: GetRestartKubeletJobName(elfMachine), 98 | Namespace: "default", 99 | }, 100 | Spec: agentv1.HostOperationJobSpec{ 101 | NodeName: elfMachine.Name, 102 | Operation: agentv1.Operation{ 103 | Ansible: &agentv1.Ansible{ 104 | LocalPlaybookText: &agentv1.YAMLText{ 105 | Inline: tasks.RestartKubeletTask, 106 | }, 107 | }, 108 | Timeout: metav1.Duration{Duration: defaultTimeout}, 109 | }, 110 | }, 111 | } 112 | } 113 | 114 | func GenerateJob(elfMachine *infrav1.ElfMachine, jobType HostAgentJobType) *agentv1.HostOperationJob { 115 | switch jobType { 116 | case HostAgentJobTypeExpandRootPartition: 117 | return GenerateExpandRootPartitionJob(elfMachine) 118 | case HostAgentJobTypeRestartKubelet: 119 | return GenerateRestartKubeletJob(elfMachine) 120 | default: 121 | return nil 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /pkg/hostagent/tasks/expand_root_partition.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Expand root partition 3 | hosts: all 4 | become: true 5 | gather_facts: true 6 | tasks: 7 | - name: Set root device 8 | ansible.builtin.set_fact: 9 | rootdevice: >- 10 | {{ ansible_mounts | selectattr('mount', 'equalto', '/') | map(attribute='device') | first }} 11 | 12 | - name: Set root partition 13 | ansible.builtin.set_fact: 14 | rootpartition: >- 15 | {{ ansible_devices 16 | | dict2items 17 | | map(attribute='value.partitions') 18 | | map('dict2items') 19 | | flatten(1) 20 | | selectattr('value.holders', 'defined') 21 | | selectattr('value.holders', 'contains', rootdevice.split('/')[-1]) 22 | | map(attribute='key') 23 | | first }} 24 | 25 | - name: Grow partition 26 | shell: | 27 | result=$(growpart /dev/{{ rootpartition | regex_replace('[0-9]+$', '') }} {{ rootpartition | regex_search('[0-9]+$') }}) 28 | if [ $? -eq 0 ]; then 29 | echo "$result" 30 | elif echo "$result" | grep -q '^NOCHANGE'; then 31 | echo "$result" 32 | else 33 | echo "$result" 34 | exit 1 35 | fi 36 | 37 | - name: Resize partition 38 | shell: pvresize /dev/{{ rootpartition }} 39 | 40 | - name: Extend root 41 | shell: | 42 | result=$(lvextend -r -l+100%FREE -n {{ rootdevice }} 2>&1) 43 | if [ $? -eq 0 ]; then 44 | echo "$result" 45 | elif echo "$result" | grep -q 'matches existing size'; then 46 | echo "$result" 47 | else 48 | echo "$result" 49 | exit 1 50 | fi 51 | -------------------------------------------------------------------------------- /pkg/hostagent/tasks/restart_kubelet.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Kubelet | restart kubelet 3 | hosts: all 4 | become: true 5 | gather_facts: false 6 | tasks: 7 | - ansible.builtin.service: 8 | name: kubelet 9 | state: restarted -------------------------------------------------------------------------------- /pkg/hostagent/tasks/tasks.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package tasks 15 | 16 | import ( 17 | _ "embed" 18 | ) 19 | 20 | // ExpandRootPartitionTask is the task to add new disk capacity to root. 21 | // 22 | //go:embed expand_root_partition.yaml 23 | var ExpandRootPartitionTask string 24 | 25 | // RestartKubeletTask is the task to restart kubelet. 26 | // 27 | //go:embed restart_kubelet.yaml 28 | var RestartKubeletTask string 29 | -------------------------------------------------------------------------------- /pkg/manager/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package manager 18 | 19 | import "time" 20 | 21 | const ( 22 | defaultPrefix = "cape-" 23 | 24 | // DefaultSyncPeriod is the default value for the eponymous 25 | // manager option. 26 | DefaultSyncPeriod = time.Minute * 10 27 | 28 | // DefaultPodName is the default value for the eponymous manager option. 29 | DefaultPodName = defaultPrefix + "controller-manager" 30 | 31 | // DefaultPodNamespace is the default value for the eponymous manager option. 32 | DefaultPodNamespace = defaultPrefix + "system" 33 | 34 | // DefaultLeaderElectionID is the default value for the eponymous manager option. 35 | DefaultLeaderElectionID = DefaultPodName + "-runtime" 36 | 37 | // DefaultWebhookServiceContainerPort is the default value for the eponymous 38 | // manager option. 39 | DefaultWebhookServiceContainerPort = 0 40 | ) 41 | -------------------------------------------------------------------------------- /pkg/manager/manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package manager 18 | 19 | import ( 20 | goctx "context" 21 | 22 | "github.com/pkg/errors" 23 | agentv1 "github.com/smartxworks/host-config-agent-api/api/v1alpha1" 24 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 25 | cgscheme "k8s.io/client-go/kubernetes/scheme" 26 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 27 | bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" 28 | controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" 29 | ctrl "sigs.k8s.io/controller-runtime" 30 | 31 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 32 | "github.com/smartxworks/cluster-api-provider-elf/pkg/context" 33 | "github.com/smartxworks/cluster-api-provider-elf/pkg/product" 34 | ) 35 | 36 | // Manager is a CAPE controller manager. 37 | type Manager interface { 38 | ctrl.Manager 39 | 40 | // GetContext returns the controller manager's context. 41 | GetControllerManagerContext() *context.ControllerManagerContext 42 | } 43 | 44 | // New returns a new CAPE controller manager. 45 | func New(ctx goctx.Context, opts Options) (Manager, error) { 46 | // Ensure the default options are set. 47 | opts.defaults() 48 | 49 | utilruntime.Must(cgscheme.AddToScheme(opts.Scheme)) 50 | utilruntime.Must(clusterv1.AddToScheme(opts.Scheme)) 51 | utilruntime.Must(infrav1.AddToScheme(opts.Scheme)) 52 | utilruntime.Must(bootstrapv1.AddToScheme(opts.Scheme)) 53 | utilruntime.Must(controlplanev1.AddToScheme(opts.Scheme)) 54 | product.InitHostAgentAPIGroup(&agentv1.GroupVersion, agentv1.SchemeBuilder) 55 | utilruntime.Must(agentv1.AddToScheme(opts.Scheme)) 56 | // +kubebuilder:scaffold:scheme 57 | 58 | // Build the controller manager. 59 | mgr, err := ctrl.NewManager(opts.KubeConfig, opts.Options) 60 | if err != nil { 61 | return nil, errors.Wrap(err, "unable to create manager") 62 | } 63 | 64 | // Build the controller manager context. 65 | ctrlMgrCtx := &context.ControllerManagerContext{ 66 | WatchNamespaces: opts.Cache.DefaultNamespaces, 67 | Namespace: opts.PodNamespace, 68 | Name: opts.PodName, 69 | LeaderElectionID: opts.LeaderElectionID, 70 | LeaderElectionNamespace: opts.LeaderElectionNamespace, 71 | Client: mgr.GetClient(), 72 | Scheme: opts.Scheme, 73 | WatchFilterValue: opts.WatchFilterValue, 74 | } 75 | 76 | // Add the requested items to the manager. 77 | if err := opts.AddToManager(ctx, ctrlMgrCtx, mgr); err != nil { 78 | return nil, errors.Wrap(err, "failed to add resources to the manager") 79 | } 80 | 81 | // +kubebuilder:scaffold:builder 82 | 83 | return &manager{ 84 | Manager: mgr, 85 | ctrlMgrCtx: ctrlMgrCtx, 86 | }, nil 87 | } 88 | 89 | type manager struct { 90 | ctrl.Manager 91 | ctrlMgrCtx *context.ControllerManagerContext 92 | } 93 | 94 | func (m *manager) GetControllerManagerContext() *context.ControllerManagerContext { 95 | return m.ctrlMgrCtx 96 | } 97 | -------------------------------------------------------------------------------- /pkg/manager/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package manager 18 | 19 | import ( 20 | goctx "context" 21 | "os" 22 | 23 | "k8s.io/apimachinery/pkg/runtime" 24 | "k8s.io/client-go/rest" 25 | "sigs.k8s.io/controller-runtime/pkg/client/config" 26 | ctrllog "sigs.k8s.io/controller-runtime/pkg/log" 27 | ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" 28 | 29 | "github.com/smartxworks/cluster-api-provider-elf/pkg/context" 30 | ) 31 | 32 | // AddToManagerFunc is a function that can be optionally specified with 33 | // the manager's Options in order to explicitly decide what controllers and 34 | // webhooks to add to the manager. 35 | type AddToManagerFunc func(goctx.Context, *context.ControllerManagerContext, ctrlmgr.Manager) error 36 | 37 | // Options describes the options used to create a new CAPE manager. 38 | type Options struct { 39 | ctrlmgr.Options 40 | 41 | // LeaderElectionNamespace is the namespace in which the pod running the 42 | // controller maintains a leader election lock 43 | // 44 | // Defaults to the eponymous constant in this package. 45 | PodNamespace string 46 | 47 | // PodName is the name of the pod running the controller manager. 48 | // 49 | // Defaults to the eponymous constant in this package. 50 | PodName string 51 | 52 | KubeConfig *rest.Config 53 | 54 | // AddToManager is a function that can be optionally specified with 55 | // the manager's Options in order to explicitly decide what controllers 56 | // and webhooks to add to the manager. 57 | AddToManager AddToManagerFunc 58 | 59 | // WatchFilterValue is used to filter incoming objects by label. 60 | // 61 | // Defaults to the empty string and by that not filter anything. 62 | WatchFilterValue string 63 | } 64 | 65 | func (o *Options) defaults() { 66 | if o.Logger.GetSink() == nil { 67 | o.Logger = ctrllog.Log 68 | } 69 | 70 | if o.PodName == "" { 71 | o.PodName = DefaultPodName 72 | } 73 | 74 | if o.KubeConfig == nil { 75 | o.KubeConfig = config.GetConfigOrDie() 76 | } 77 | 78 | if o.Scheme == nil { 79 | o.Scheme = runtime.NewScheme() 80 | } 81 | 82 | if ns, ok := os.LookupEnv("POD_NAMESPACE"); ok { 83 | o.PodNamespace = ns 84 | } else { 85 | o.PodNamespace = DefaultPodNamespace 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /pkg/product/product.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package product 18 | 19 | import ( 20 | "os" 21 | 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | "sigs.k8s.io/controller-runtime/pkg/scheme" 24 | ) 25 | 26 | func InitHostAgentAPIGroup(groupVersion *schema.GroupVersion, schemeBuilder *scheme.Builder) { 27 | hostAgentAPIGroup := os.Getenv("HOST_CONFIG_AGENT_API_GROUP") 28 | if hostAgentAPIGroup != "" { 29 | groupVersion.Group = hostAgentAPIGroup 30 | schemeBuilder.GroupVersion = *groupVersion 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/resources/label.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package resources 18 | 19 | import ( 20 | "fmt" 21 | ) 22 | 23 | // Tower labels for VMs created by CAPE. 24 | const ( 25 | VMLabelManaged = "managed" 26 | VMLabelNamespace = "namespace" 27 | VMLabelClusterName = "cluster-name" 28 | VMLabelVIP = "vip" 29 | ) 30 | 31 | func GetVMLabelManaged() string { 32 | return fmt.Sprintf("%s-%s", GetResourcePrefix(), VMLabelManaged) 33 | } 34 | 35 | func GetVMLabelNamespace() string { 36 | return fmt.Sprintf("%s-%s", GetResourcePrefix(), VMLabelNamespace) 37 | } 38 | 39 | func GetVMLabelClusterName() string { 40 | return fmt.Sprintf("%s-%s", GetResourcePrefix(), VMLabelClusterName) 41 | } 42 | 43 | func GetVMLabelVIP() string { 44 | return fmt.Sprintf("%s-%s", GetResourcePrefix(), VMLabelVIP) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/resources/placement_group.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package resources 18 | 19 | import ( 20 | goctx "context" 21 | "fmt" 22 | 23 | "github.com/smartxworks/cloudtower-go-sdk/v2/models" 24 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 25 | "sigs.k8s.io/controller-runtime/pkg/client" 26 | 27 | annotationsutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/annotations" 28 | labelsutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/labels" 29 | machineutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/machine" 30 | ) 31 | 32 | func GetVMPlacementGroupName(ctx goctx.Context, ctrlClient client.Client, machine *clusterv1.Machine, cluster *clusterv1.Cluster) (string, error) { 33 | groupName := "" 34 | if machineutil.IsControlPlaneMachine(machine) { 35 | kcp, err := machineutil.GetKCPByMachine(ctx, ctrlClient, machine) 36 | if err != nil { 37 | return "", err 38 | } 39 | 40 | placementGroupName := annotationsutil.GetPlacementGroupName(kcp) 41 | if placementGroupName != "" { 42 | return placementGroupName, nil 43 | } 44 | 45 | groupName = machineutil.GetKCPNameByMachine(machine) 46 | } else { 47 | md, err := machineutil.GetMDByMachine(ctx, ctrlClient, machine) 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | placementGroupName := annotationsutil.GetPlacementGroupName(md) 53 | if placementGroupName != "" { 54 | return placementGroupName, nil 55 | } 56 | 57 | groupName = labelsutil.GetDeploymentNameLabel(machine) 58 | } 59 | 60 | if groupName == "" { 61 | return "", nil 62 | } 63 | 64 | return fmt.Sprintf("%s-%s", GetVMPlacementGroupNamePrefix(cluster), groupName), nil 65 | } 66 | 67 | func GetVMPlacementGroupNamePrefix(cluster *clusterv1.Cluster) string { 68 | return fmt.Sprintf("%s-managed-%s-%s", GetResourcePrefix(), cluster.UID, cluster.Namespace) 69 | } 70 | 71 | func GetVMPlacementGroupPolicy(machine *clusterv1.Machine) models.VMVMPolicy { 72 | if machineutil.IsControlPlaneMachine(machine) { 73 | return models.VMVMPolicyMUSTDIFFERENT 74 | } 75 | 76 | return models.VMVMPolicyPREFERDIFFERENT 77 | } 78 | -------------------------------------------------------------------------------- /pkg/resources/resource.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package resources 18 | 19 | import "github.com/smartxworks/cluster-api-provider-elf/pkg/util" 20 | 21 | // Tower resources. 22 | const ( 23 | TowerResourcePrefix = "TOWER_RESOURCE_PREFIX" 24 | 25 | // By default, CAPE allow modify the configuration(CPU) of VM directly. 26 | // If you want to limit the VM configuration to be modified directly, 27 | // set AllowCustomVMConfig to false, CAPE will try to restore the VM configuration 28 | // set by ElfMachine. 29 | AllowCustomVMConfig = "ALLOW_CUSTOM_VM_CONFIG" 30 | ) 31 | 32 | func GetResourcePrefix() string { 33 | return util.GetEnv(TowerResourcePrefix, "cape") 34 | } 35 | 36 | func IsAllowCustomVMConfig() bool { 37 | return util.GetEnv(AllowCustomVMConfig, "true") == "false" 38 | } 39 | -------------------------------------------------------------------------------- /pkg/service/consts.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package service 18 | 19 | const ( 20 | // VMPlacementGroupDescription is the description of vm placement group. 21 | VMPlacementGroupDescription = "This is VM placement group created by CAPE, don't delete it!" 22 | 23 | // SKSVMTemplateUIDLabel is the label used to find the virtual machine template. 24 | SKSVMTemplateUIDLabel = "system.cloudtower/sks-template-uid" 25 | ) 26 | 27 | // VMOwnerSearchForUsername is used to specify the ower source of the virtual machine. 28 | // TODO: Tower SDK will provide the VMOwnerSearchFor enumeration types in the new version. 29 | const VMOwnerSearchForUsername = "username" 30 | -------------------------------------------------------------------------------- /pkg/service/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package service 18 | 19 | import "fmt" 20 | 21 | type GPUDeviceInfo struct { 22 | ID string `json:"id"` 23 | HostID string `json:"hostId"` 24 | // AllocatedCount the number that has been allocated. 25 | AllocatedCount int32 `json:"allocatedCount"` 26 | // AvailableCount is the number of GPU that can be allocated. 27 | AvailableCount int32 `json:"availableCount"` 28 | } 29 | 30 | func (g *GPUDeviceInfo) String() string { 31 | return fmt.Sprintf("{id:%s, hostId:%s, allocatedCount:%d, availableCount:%d}", g.ID, g.HostID, g.AllocatedCount, g.AvailableCount) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/session/tower.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package session 18 | 19 | import ( 20 | goctx "context" 21 | "crypto/sha256" 22 | "encoding/hex" 23 | "fmt" 24 | "sync" 25 | "time" 26 | 27 | "github.com/go-logr/logr" 28 | httptransport "github.com/go-openapi/runtime/client" 29 | "github.com/go-openapi/strfmt" 30 | "github.com/pkg/errors" 31 | towerclient "github.com/smartxworks/cloudtower-go-sdk/v2/client" 32 | "github.com/smartxworks/cloudtower-go-sdk/v2/client/user" 33 | "github.com/smartxworks/cloudtower-go-sdk/v2/models" 34 | ctrl "sigs.k8s.io/controller-runtime" 35 | 36 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 37 | ) 38 | 39 | var lastGCTime = time.Now() 40 | var gcMinInterval = 10 * time.Minute 41 | var sessionIdleTime = 10 * time.Minute 42 | 43 | // global Session map against sessionKeys 44 | // in map[sessionKey]cacheItem. 45 | var sessionCache sync.Map 46 | 47 | type cacheItem struct { 48 | LastUsedTime time.Time 49 | Session *TowerSession 50 | } 51 | 52 | type TowerSession struct { 53 | *towerclient.Cloudtower 54 | } 55 | 56 | // GetOrCreate gets a cached session or creates a new one if one does not 57 | // already exist. 58 | func GetOrCreate(ctx goctx.Context, tower infrav1.Tower) (*TowerSession, error) { 59 | logger := ctrl.LoggerFrom(ctx).WithName("session").WithValues("server", tower.Server, "username", tower.Username, "source", tower.AuthMode) 60 | 61 | defer func() { 62 | if lastGCTime.Add(gcMinInterval).Before(time.Now()) { 63 | cleanupSessionCache(logger) 64 | lastGCTime = time.Now() 65 | } 66 | }() 67 | 68 | sessionKey := getSessionKey(&tower) 69 | if value, ok := sessionCache.Load(sessionKey); ok { 70 | item := value.(*cacheItem) 71 | item.LastUsedTime = time.Now() 72 | logger.V(3).Info("found active cached tower client session") 73 | 74 | return item.Session, nil 75 | } 76 | 77 | client, err := createTowerClient(httptransport.TLSClientOptions{ 78 | InsecureSkipVerify: tower.SkipTLSVerify, 79 | }, towerclient.ClientConfig{ 80 | Host: tower.Server, 81 | BasePath: "/v2/api", 82 | Schemes: []string{"https"}, 83 | }, towerclient.UserConfig{ 84 | Name: tower.Username, 85 | Password: tower.Password, 86 | Source: models.UserSource(tower.AuthMode), 87 | }) 88 | if err != nil { 89 | return nil, errors.Wrap(err, "failed to create tower client") 90 | } 91 | 92 | session := &TowerSession{client} 93 | 94 | // Cache the session. 95 | sessionCache.Store(sessionKey, &cacheItem{LastUsedTime: time.Now(), Session: session}) 96 | logger.V(3).Info("cached tower client session") 97 | 98 | return session, nil 99 | } 100 | 101 | func createTowerClient(tlsOpts httptransport.TLSClientOptions, clientConfig towerclient.ClientConfig, userConfig towerclient.UserConfig) (*towerclient.Cloudtower, error) { 102 | rt := httptransport.New(clientConfig.Host, clientConfig.BasePath, clientConfig.Schemes) 103 | var err error 104 | rt.Transport, err = httptransport.TLSTransport(tlsOpts) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | client := towerclient.New(rt, strfmt.Default) 110 | params := user.NewLoginParams() 111 | params.RequestBody = &models.LoginInput{ 112 | Username: &userConfig.Name, 113 | Password: &userConfig.Password, 114 | Source: userConfig.Source.Pointer(), 115 | } 116 | resp, err := client.User.Login(params) 117 | if err != nil { 118 | return nil, err 119 | } 120 | rt.DefaultAuthentication = httptransport.APIKeyAuth("Authorization", "header", *resp.Payload.Data.Token) 121 | return client, nil 122 | } 123 | 124 | func cleanupSessionCache(logger logr.Logger) { 125 | sessionCache.Range(func(key interface{}, value interface{}) bool { 126 | item := value.(*cacheItem) 127 | if item.LastUsedTime.Add(sessionIdleTime).Before(time.Now()) { 128 | sessionCache.Delete(key) 129 | logger.V(3).Info(fmt.Sprintf("delete inactive tower client session %s from sessionCache", key)) 130 | } 131 | 132 | return true 133 | }) 134 | } 135 | 136 | func getSessionKey(tower *infrav1.Tower) string { 137 | encryptedTower := tower.DeepCopy() 138 | sum256 := sha256.Sum256([]byte(tower.Password)) 139 | encryptedTower.Password = hex.EncodeToString(sum256[:]) 140 | 141 | return fmt.Sprintf("%v", encryptedTower) 142 | } 143 | -------------------------------------------------------------------------------- /pkg/session/tower_test.go: -------------------------------------------------------------------------------- 1 | package session 2 | 3 | import ( 4 | goctx "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/onsi/gomega" 9 | 10 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 11 | ) 12 | 13 | func TestGetOrCreate(t *testing.T) { 14 | g := gomega.NewGomegaWithT(t) 15 | 16 | t.Run("should get cached session and clear inactive session", func(t *testing.T) { 17 | lastGCTime = lastGCTime.Add(-gcMinInterval) 18 | tower := infrav1.Tower{Server: "127.0.0.1", Username: "tower", Password: "tower"} 19 | inactiveTower := tower.DeepCopy() 20 | inactiveTower.Username = "inactive" 21 | invalidTower := tower.DeepCopy() 22 | invalidTower.Username = "invalid" 23 | 24 | sessionKey := getSessionKey(&tower) 25 | cachedSession := &TowerSession{} 26 | sessionCache.Store(sessionKey, &cacheItem{Session: cachedSession}) 27 | inactiveSessionKey := getSessionKey(inactiveTower) 28 | sessionCache.Store(inactiveSessionKey, &cacheItem{Session: &TowerSession{}, LastUsedTime: time.Now().Add(-sessionIdleTime)}) 29 | 30 | session, err := GetOrCreate(goctx.Background(), tower) 31 | g.Expect(err).ToNot(gomega.HaveOccurred()) 32 | g.Expect(session).To(gomega.Equal(cachedSession)) 33 | 34 | _, ok := sessionCache.Load(inactiveSessionKey) 35 | g.Expect(ok).To(gomega.BeFalse()) 36 | 37 | session, err = GetOrCreate(goctx.Background(), *invalidTower) 38 | g.Expect(session).To(gomega.BeNil()) 39 | g.Expect(err).To(gomega.HaveOccurred()) 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/util/annotations/helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package annotations 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 22 | "sigs.k8s.io/cluster-api/util/annotations" 23 | 24 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 25 | ) 26 | 27 | // HasAnnotation returns true if the object has the specified annotation. 28 | func HasAnnotation(o metav1.Object, annotationKey string) bool { 29 | annotations := o.GetAnnotations() 30 | if annotations == nil { 31 | return false 32 | } 33 | 34 | _, ok := annotations[annotationKey] 35 | return ok 36 | } 37 | 38 | func GetPlacementGroupName(o metav1.Object) string { 39 | annotations := o.GetAnnotations() 40 | if annotations == nil { 41 | return "" 42 | } 43 | 44 | return annotations[infrav1.PlacementGroupNameAnnotation] 45 | } 46 | 47 | func GetCreatedBy(o metav1.Object) string { 48 | annotations := o.GetAnnotations() 49 | if annotations == nil { 50 | return "" 51 | } 52 | 53 | return annotations[infrav1.CreatedByAnnotation] 54 | } 55 | 56 | func GetTemplateClonedFromName(o metav1.Object) string { 57 | annotations := o.GetAnnotations() 58 | if annotations == nil { 59 | return "" 60 | } 61 | 62 | return annotations[clusterv1.TemplateClonedFromNameAnnotation] 63 | } 64 | 65 | // AddAnnotations sets the desired annotations on the object and returns true if the annotations have changed. 66 | func AddAnnotations(o metav1.Object, desired map[string]string) bool { 67 | return annotations.AddAnnotations(o, desired) 68 | } 69 | 70 | // RemoveAnnotation deletes the desired annotation on the object. 71 | func RemoveAnnotation(o metav1.Object, annotation string) { 72 | annotations := o.GetAnnotations() 73 | if annotations == nil { 74 | return 75 | } 76 | delete(annotations, annotation) 77 | o.SetAnnotations(annotations) 78 | } 79 | -------------------------------------------------------------------------------- /pkg/util/common.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "os" 4 | 5 | func GetEnv(envKey, defaultValue string) string { 6 | if value, ok := os.LookupEnv(envKey); ok { 7 | return value 8 | } 9 | 10 | return defaultValue 11 | } 12 | -------------------------------------------------------------------------------- /pkg/util/kcp/kcp.go: -------------------------------------------------------------------------------- 1 | package kcp 2 | 3 | /* 4 | Copyright 2023. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | import ( 20 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 21 | controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" 22 | "sigs.k8s.io/cluster-api/util/conditions" 23 | ) 24 | 25 | // IsKCPInRollingUpdate returns whether KCP is in rolling update. 26 | // 27 | // When *kcp.Spec.Replicas > kcp.Status.UpdatedReplicas, it must be in a KCP rolling update process. 28 | // When *kcp.Spec.Replicas == kcp.Status.UpdatedReplicas, it could be in one of the following cases: 29 | // 1. It's not in a KCP rolling update process. So kcp.Spec.Replicas == kcp.Status.Replicas. 30 | // 2. It's at the end of a KCP rolling update process, and the last KCP replica (i.e the last KCP ElfMachine) is created just now. 31 | // There is still an old KCP ElfMachine, so kcp.Spec.Replicas + 1 == kcp.Status.Replicas. 32 | // 33 | // For more information about KCP replicas, refer to https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go 34 | // For more information about KCP rollout, refer to https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20191017-kubeadm-based-control-plane.md#kubeadmcontrolplane-rollout 35 | func IsKCPInRollingUpdate(kcp *controlplanev1.KubeadmControlPlane) bool { 36 | if (*kcp.Spec.Replicas > kcp.Status.UpdatedReplicas && *kcp.Spec.Replicas <= kcp.Status.Replicas) || 37 | (*kcp.Spec.Replicas == kcp.Status.UpdatedReplicas && *kcp.Spec.Replicas < kcp.Status.Replicas) { 38 | return true 39 | } 40 | 41 | return false 42 | } 43 | 44 | // IsKCPInScalingDown returns whether KCP is in scaling down. 45 | // 46 | // When KCP is in scaling down/rolling update, KCP controller marks 47 | // ResizedCondition to false and ScalingDownReason as Reason. 48 | // 49 | // For more information about KCP ResizedCondition and ScalingDownReason, refer to https://github.com/kubernetes-sigs/cluster-api/blob/main/api/v1beta1/condition_consts.go 50 | func IsKCPInScalingDown(kcp *controlplanev1.KubeadmControlPlane) bool { 51 | return conditions.IsFalse(kcp, clusterv1.ResizedCondition) && 52 | conditions.GetReason(kcp, controlplanev1.ResizedCondition) == controlplanev1.ScalingDownReason && 53 | *kcp.Spec.Replicas < kcp.Status.UpdatedReplicas 54 | } 55 | -------------------------------------------------------------------------------- /pkg/util/kube.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | "github.com/pkg/errors" 24 | "k8s.io/client-go/kubernetes" 25 | "k8s.io/client-go/tools/clientcmd" 26 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 27 | "sigs.k8s.io/cluster-api/util/kubeconfig" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | ) 30 | 31 | // NewKubeClient returns a new client for the target cluster using the KubeConfig 32 | // secret stored in the management cluster. 33 | func NewKubeClient( 34 | ctx context.Context, 35 | controllerClient client.Client, 36 | cluster *clusterv1.Cluster) (kubernetes.Interface, error) { 37 | clusterKey := client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Name} 38 | kubeconfig, err := kubeconfig.FromSecret(ctx, controllerClient, clusterKey) 39 | if err != nil { 40 | return nil, errors.Wrapf(err, "failed to retrieve kubeconfig secret for Cluster %q in namespace %q", 41 | cluster.Name, cluster.Namespace) 42 | } 43 | 44 | restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) 45 | if err != nil { 46 | return nil, errors.Wrapf(err, "failed to create client configuration for Cluster %q in namespace %q", 47 | cluster.Name, cluster.Namespace) 48 | } 49 | // sets the timeout, otherwise this will default to 0 (i.e. no timeout) which might cause tests to hang 50 | restConfig.Timeout = 10 * time.Second 51 | 52 | return kubernetes.NewForConfig(restConfig) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/util/labels/helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package labels 18 | 19 | import ( 20 | "regexp" 21 | 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "k8s.io/apimachinery/pkg/util/validation" 24 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 25 | 26 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 27 | ) 28 | 29 | const ( 30 | // ClusterAutoscalerCAPIGPULabel is the label added to nodes with GPU resource, which will be used by clusterAutoscaler-CAPI. 31 | // ref: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#special-note-on-gpu-instances 32 | ClusterAutoscalerCAPIGPULabel = "cluster-api/accelerator" 33 | ) 34 | 35 | var ( 36 | noLabelCharReg = regexp.MustCompile(`[^-a-zA-Z0-9-.]`) 37 | ) 38 | 39 | func GetHostServerIDLabel(o metav1.Object) string { 40 | return GetLabelValue(o, infrav1.HostServerIDLabel) 41 | } 42 | 43 | func GetHostServerNameLabel(o metav1.Object) string { 44 | return GetLabelValue(o, infrav1.HostServerNameLabel) 45 | } 46 | 47 | func GetTowerVMIDLabel(o metav1.Object) string { 48 | return GetLabelValue(o, infrav1.TowerVMIDLabel) 49 | } 50 | 51 | func GetNodeGroupLabel(o metav1.Object) string { 52 | return GetLabelValue(o, infrav1.NodeGroupLabel) 53 | } 54 | 55 | func GetClusterNameLabelLabel(o metav1.Object) string { 56 | return GetLabelValue(o, clusterv1.ClusterNameLabel) 57 | } 58 | 59 | func GetDeploymentNameLabel(o metav1.Object) string { 60 | return GetLabelValue(o, clusterv1.MachineDeploymentNameLabel) 61 | } 62 | 63 | func GetClusterNameLabel(o metav1.Object) string { 64 | return GetLabelValue(o, clusterv1.ClusterNameLabel) 65 | } 66 | 67 | func GetLabelValue(o metav1.Object, label string) string { 68 | labels := o.GetLabels() 69 | if labels == nil { 70 | return "" 71 | } 72 | 73 | val, ok := labels[label] 74 | if !ok { 75 | return "" 76 | } 77 | 78 | return val 79 | } 80 | 81 | // ConvertToLabelValue converts a string to a valid label value. 82 | // A valid label value must be an empty string or consist of alphanumeric characters, '-', '_' or '.'. 83 | func ConvertToLabelValue(str string) string { 84 | result := noLabelCharReg.ReplaceAll([]byte(str), []byte("-")) 85 | 86 | if len(result) > validation.LabelValueMaxLength { 87 | return string(result[:validation.LabelValueMaxLength]) 88 | } 89 | 90 | for len(result) > 0 && (result[0] == '-' || result[0] == '_' || result[0] == '.') { 91 | result = result[1:] 92 | } 93 | for len(result) > 0 && (result[len(result)-1] == '-' || result[len(result)-1] == '_' || result[len(result)-1] == '.') { 94 | result = result[:len(result)-1] 95 | } 96 | 97 | return string(result) 98 | } 99 | -------------------------------------------------------------------------------- /pkg/util/labels/helpers_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package labels 18 | 19 | import ( 20 | "strconv" 21 | "testing" 22 | 23 | "github.com/onsi/gomega" 24 | ) 25 | 26 | func TestConvertToLabelValue(t *testing.T) { 27 | g := gomega.NewGomegaWithT(t) 28 | 29 | testCases := []struct { 30 | str string 31 | labelValue string 32 | }{ 33 | { 34 | str: "test", 35 | labelValue: "test", 36 | }, { 37 | str: "Test", 38 | labelValue: "Test", 39 | }, { 40 | str: "TEST", 41 | labelValue: "TEST", 42 | }, { 43 | str: "Tesla V100-PCIE-16GB", 44 | labelValue: "Tesla-V100-PCIE-16GB", 45 | }, { 46 | str: "Tesla T4", 47 | labelValue: "Tesla-T4", 48 | }, { 49 | str: ".test.", 50 | labelValue: "test", 51 | }, { 52 | str: "-test-", 53 | labelValue: "test", 54 | }, { 55 | str: "_test_", 56 | labelValue: "test", 57 | }, { 58 | str: "_test_", 59 | labelValue: "test", 60 | }, { 61 | str: "!!! te st !!!", 62 | labelValue: "te-st", 63 | }, { 64 | str: "toolongtoolongtoolongtoolongtoolongtoolongtoolongtoolongtoolongtoolong", 65 | labelValue: "toolongtoolongtoolongtoolongtoolongtoolongtoolongtoolongtoolong", 66 | }, 67 | } 68 | 69 | for i, tc := range testCases { 70 | t.Run(strconv.Itoa(i), func(t *testing.T) { 71 | labelValue := ConvertToLabelValue(tc.str) 72 | g.Expect(labelValue).To(gomega.Equal(tc.labelValue)) 73 | }) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/util/machine/kcp.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package machine 18 | 19 | import ( 20 | goctx "context" 21 | "fmt" 22 | 23 | apitypes "k8s.io/apimachinery/pkg/types" 24 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 25 | controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | ) 28 | 29 | // GetKCPNameByMachine returns the KCP name associated with the Machine. 30 | // Do not use "cluster.x-k8s.io/control-plane-name" label because 31 | // its value will be a hashed string of the KCP name when the KCP name exceeds 63 characters. 32 | func GetKCPNameByMachine(machine *clusterv1.Machine) string { 33 | for _, o := range machine.OwnerReferences { 34 | if o.Kind == "KubeadmControlPlane" { 35 | return o.Name 36 | } 37 | } 38 | panic(fmt.Sprintf("Machine %s is not owned by KubeadmControlPlane", machine.GetName())) 39 | } 40 | 41 | func GetKCPByMachine(ctx goctx.Context, ctrlClient client.Client, machine *clusterv1.Machine) (*controlplanev1.KubeadmControlPlane, error) { 42 | var kcp controlplanev1.KubeadmControlPlane 43 | 44 | kcpName := GetKCPNameByMachine(machine) 45 | if err := ctrlClient.Get(ctx, apitypes.NamespacedName{Namespace: machine.Namespace, Name: kcpName}, &kcp); err != nil { 46 | return nil, err 47 | } 48 | 49 | return &kcp, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/util/machine/kcp_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package machine 18 | 19 | import ( 20 | goctx "context" 21 | "testing" 22 | 23 | "github.com/onsi/gomega" 24 | 25 | "github.com/smartxworks/cluster-api-provider-elf/test/fake" 26 | ) 27 | 28 | func TestGetKCPByMachine(t *testing.T) { 29 | g := gomega.NewGomegaWithT(t) 30 | ctx := goctx.TODO() 31 | elfCluster, cluster := fake.NewClusterObjects() 32 | _, cpMachine := fake.NewMachineObjects(elfCluster, cluster) 33 | kubeadmCP := fake.NewKCP() 34 | fake.ToCPMachine(cpMachine, kubeadmCP) 35 | ctrlMgrCtx := fake.NewControllerManagerContext(kubeadmCP) 36 | t.Run("should return kcp", func(t *testing.T) { 37 | kcp, err := GetKCPByMachine(ctx, ctrlMgrCtx.Client, cpMachine) 38 | g.Expect(err).ToNot(gomega.HaveOccurred()) 39 | g.Expect(kcp.Name).To(gomega.Equal(kubeadmCP.Name)) 40 | }) 41 | 42 | _, workerMachine := fake.NewMachineObjects(elfCluster, cluster) 43 | t.Run("should panic when failed to get kcp name", func(t *testing.T) { 44 | g.Expect(func() { _, _ = GetKCPByMachine(ctx, ctrlMgrCtx.Client, workerMachine) }).To(gomega.Panic()) 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/util/machine/md.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package machine 18 | 19 | import ( 20 | goctx "context" 21 | 22 | apitypes "k8s.io/apimachinery/pkg/types" 23 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 24 | "sigs.k8s.io/controller-runtime/pkg/client" 25 | 26 | labelutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/labels" 27 | ) 28 | 29 | func GetMDByMachine(ctx goctx.Context, ctrlClient client.Client, machine *clusterv1.Machine) (*clusterv1.MachineDeployment, error) { 30 | var md clusterv1.MachineDeployment 31 | if err := ctrlClient.Get(ctx, apitypes.NamespacedName{Namespace: machine.Namespace, Name: labelutil.GetDeploymentNameLabel(machine)}, &md); err != nil { 32 | return nil, err 33 | } 34 | 35 | return &md, nil 36 | } 37 | 38 | func GetMDsForCluster( 39 | ctx goctx.Context, 40 | ctrlClient client.Client, 41 | namespace, clusterName string) ([]*clusterv1.MachineDeployment, error) { 42 | var mdList clusterv1.MachineDeploymentList 43 | labels := map[string]string{clusterv1.ClusterNameLabel: clusterName} 44 | 45 | if err := ctrlClient.List( 46 | ctx, &mdList, 47 | client.InNamespace(namespace), 48 | client.MatchingLabels(labels)); err != nil { 49 | return nil, err 50 | } 51 | 52 | mds := make([]*clusterv1.MachineDeployment, len(mdList.Items)) 53 | for i := range mdList.Items { 54 | mds[i] = &mdList.Items[i] 55 | } 56 | 57 | return mds, nil 58 | } 59 | -------------------------------------------------------------------------------- /pkg/util/machine/md_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package machine 18 | 19 | import ( 20 | goctx "context" 21 | "testing" 22 | 23 | "github.com/onsi/gomega" 24 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 25 | 26 | "github.com/smartxworks/cluster-api-provider-elf/test/fake" 27 | ) 28 | 29 | func TestGetMDByMachine(t *testing.T) { 30 | g := gomega.NewGomegaWithT(t) 31 | ctx := goctx.TODO() 32 | elfCluster, cluster := fake.NewClusterObjects() 33 | _, machine := fake.NewMachineObjects(elfCluster, cluster) 34 | machineDeployment := fake.NewMD() 35 | fake.ToWorkerMachine(machine, machineDeployment) 36 | ctrlMgrCtx := fake.NewControllerManagerContext(machineDeployment) 37 | 38 | t.Run("should return md", func(t *testing.T) { 39 | md, err := GetMDByMachine(ctx, ctrlMgrCtx.Client, machine) 40 | g.Expect(err).ToNot(gomega.HaveOccurred()) 41 | g.Expect(md.Name).To(gomega.Equal(machineDeployment.Name)) 42 | }) 43 | } 44 | 45 | func TestGetMDsForCluster(t *testing.T) { 46 | g := gomega.NewGomegaWithT(t) 47 | ctx := goctx.TODO() 48 | _, cluster := fake.NewClusterObjects() 49 | md1 := fake.NewMD() 50 | md1.Labels = map[string]string{clusterv1.ClusterNameLabel: cluster.Name} 51 | md2 := fake.NewMD() 52 | ctrlMgrCtx := fake.NewControllerManagerContext(md1, md2) 53 | 54 | t.Run("should return mds", func(t *testing.T) { 55 | mds, err := GetMDsForCluster(ctx, ctrlMgrCtx.Client, cluster.Namespace, cluster.Name) 56 | g.Expect(err).ToNot(gomega.HaveOccurred()) 57 | g.Expect(mds).To(gomega.HaveLen(1)) 58 | g.Expect(mds[0].Name).To(gomega.Equal(md1.Name)) 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /pkg/util/md/md.go: -------------------------------------------------------------------------------- 1 | package md 2 | 3 | /* 4 | Copyright 2024. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | import ( 20 | intstrutil "k8s.io/apimachinery/pkg/util/intstr" 21 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 22 | ) 23 | 24 | // IsMDInRollingUpdate returns whether MD is in rolling update. 25 | // 26 | // When *md.Spec.Replicas > md.Status.UpdatedReplicas, it must be in a MD rolling update process. 27 | // When *md.Spec.Replicas == md.Status.UpdatedReplicas, it could be in one of the following cases: 28 | // 1. It's not in a MD rolling update process. So md.Spec.Replicas == md.Status.Replicas. 29 | // 2. It's at the end of a MD rolling update process, and the last MD replica (i.e the last MD ElfMachine) is created just now. 30 | // There is still an old MD ElfMachine, so md.Spec.Replicas + 1 == md.Status.Replicas. 31 | func IsMDInRollingUpdate(md *clusterv1.MachineDeployment) bool { 32 | if (*md.Spec.Replicas > md.Status.UpdatedReplicas && *md.Spec.Replicas <= md.Status.Replicas) || 33 | (*md.Spec.Replicas == md.Status.UpdatedReplicas && *md.Spec.Replicas < md.Status.Replicas) { 34 | return true 35 | } 36 | 37 | return false 38 | } 39 | 40 | /* 41 | Copy from CAPI: https://github.com/kubernetes-sigs/cluster-api/blob/release-1.5/internal/controllers/machinedeployment/mdutil/util.go 42 | */ 43 | 44 | // MaxUnavailable returns the maximum unavailable machines a rolling deployment can take. 45 | func MaxUnavailable(deployment clusterv1.MachineDeployment) int32 { 46 | if deployment.Spec.Strategy.Type != clusterv1.RollingUpdateMachineDeploymentStrategyType || *(deployment.Spec.Replicas) == 0 { 47 | return int32(0) 48 | } 49 | // Error caught by validation 50 | _, maxUnavailable, _ := ResolveFenceposts(deployment.Spec.Strategy.RollingUpdate.MaxSurge, deployment.Spec.Strategy.RollingUpdate.MaxUnavailable, *(deployment.Spec.Replicas)) 51 | if maxUnavailable > *deployment.Spec.Replicas { 52 | return *deployment.Spec.Replicas 53 | } 54 | return maxUnavailable 55 | } 56 | 57 | // MaxSurge returns the maximum surge machines a rolling deployment can take. 58 | func MaxSurge(deployment clusterv1.MachineDeployment) int32 { 59 | if deployment.Spec.Strategy.Type != clusterv1.RollingUpdateMachineDeploymentStrategyType { 60 | return int32(0) 61 | } 62 | // Error caught by validation 63 | maxSurge, _, _ := ResolveFenceposts(deployment.Spec.Strategy.RollingUpdate.MaxSurge, deployment.Spec.Strategy.RollingUpdate.MaxUnavailable, *(deployment.Spec.Replicas)) 64 | return maxSurge 65 | } 66 | 67 | // ResolveFenceposts resolves both maxSurge and maxUnavailable. This needs to happen in one 68 | // step. For example: 69 | // 70 | // 2 desired, max unavailable 1%, surge 0% - should scale old(-1), then new(+1), then old(-1), then new(+1) 71 | // 1 desired, max unavailable 1%, surge 0% - should scale old(-1), then new(+1) 72 | // 2 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1) 73 | // 1 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1) 74 | // 2 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1) 75 | // 1 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1). 76 | func ResolveFenceposts(maxSurge, maxUnavailable *intstrutil.IntOrString, desired int32) (int32, int32, error) { 77 | surge, err := intstrutil.GetScaledValueFromIntOrPercent(maxSurge, int(desired), true) 78 | if err != nil { 79 | return 0, 0, err 80 | } 81 | unavailable, err := intstrutil.GetScaledValueFromIntOrPercent(maxUnavailable, int(desired), false) 82 | if err != nil { 83 | return 0, 0, err 84 | } 85 | 86 | if surge == 0 && unavailable == 0 { 87 | // Validation should never allow the user to explicitly use zero values for both maxSurge 88 | // maxUnavailable. Due to rounding down maxUnavailable though, it may resolve to zero. 89 | // If both fenceposts resolve to zero, then we should set maxUnavailable to 1 on the 90 | // theory that surge might not work due to quota. 91 | unavailable = 1 92 | } 93 | 94 | return int32(surge), int32(unavailable), nil //nolint:gosec 95 | } 96 | -------------------------------------------------------------------------------- /pkg/util/patch/finalizer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package patch 18 | 19 | import ( 20 | goctx "context" 21 | 22 | "sigs.k8s.io/controller-runtime/pkg/client" 23 | ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 24 | ) 25 | 26 | // AddFinalizerWithOptimisticLock adds a finalizer to the specified object 27 | // and saves it by using resourceVersion optimistic lock. 28 | func AddFinalizerWithOptimisticLock(ctx goctx.Context, c client.Client, obj client.Object, finalizer string) error { 29 | modifiedObj := obj.DeepCopyObject().(client.Object) 30 | ctrlutil.AddFinalizer(modifiedObj, finalizer) 31 | return c.Patch(ctx, modifiedObj, client.MergeFromWithOptions(obj, client.MergeFromWithOptimisticLock{})) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/util/tower.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "github.com/smartxworks/cloudtower-go-sdk/v2/models" 21 | 22 | "github.com/smartxworks/cluster-api-provider-elf/pkg/service" 23 | typesutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/types" 24 | ) 25 | 26 | // GetVMRef returns the ID or localID of the VM. 27 | // If the localID is in UUID format, return the localID, otherwise return the ID. 28 | // 29 | // Before the ELF VM is created, Tower sets a "placeholder-{UUID}" format string to localID, such as "placeholder-7d8b6df1-c623-4750-a771-3ba6b46995fa". 30 | // After the ELF VM is created, Tower sets the VM ID in UUID format to localID. 31 | func GetVMRef(vm *models.VM) string { 32 | if vm == nil { 33 | return "" 34 | } 35 | 36 | vmLocalID := service.GetTowerString(vm.LocalID) 37 | if typesutil.IsUUID(vmLocalID) { 38 | return vmLocalID 39 | } 40 | 41 | return *vm.ID 42 | } 43 | -------------------------------------------------------------------------------- /pkg/util/types/uuid.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package types 18 | 19 | import "regexp" 20 | 21 | const ( 22 | // UUIDPattern is a regex pattern and is used by ConvertUUIDToProviderID 23 | // to convert a UUID into a providerID string. 24 | UUIDPattern = `(?i)^[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}$` 25 | ) 26 | 27 | func IsUUID(uuid string) bool { 28 | if uuid == "" { 29 | return false 30 | } 31 | 32 | pattern := regexp.MustCompile(UUIDPattern) 33 | 34 | return pattern.MatchString(uuid) 35 | } 36 | -------------------------------------------------------------------------------- /pkg/util/types/uuid_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package types 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/onsi/gomega" 23 | ) 24 | 25 | func TestIsUUID(t *testing.T) { 26 | g := gomega.NewGomegaWithT(t) 27 | 28 | testCases := []struct { 29 | name string 30 | uuid string 31 | isUUID bool 32 | }{ 33 | { 34 | name: "empty uuid", 35 | uuid: "", 36 | isUUID: false, 37 | }, 38 | { 39 | name: "invalid uuid", 40 | uuid: "1234", 41 | isUUID: false, 42 | }, 43 | { 44 | name: "valid uuid", 45 | uuid: "12345678-1234-1234-1234-123456789abc", 46 | isUUID: true, 47 | }, 48 | { 49 | name: "mixed case", 50 | uuid: "12345678-1234-1234-1234-123456789AbC", 51 | isUUID: true, 52 | }, 53 | { 54 | name: "invalid hex chars", 55 | uuid: "12345678-1234-1234-1234-123456789abg", 56 | isUUID: false, 57 | }, 58 | } 59 | 60 | for _, tc := range testCases { 61 | t.Run(tc.name, func(t *testing.T) { 62 | isUUID := IsUUID(tc.uuid) 63 | g.Expect(isUUID).To(gomega.Equal(tc.isUUID)) 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/version/consts.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package version 18 | 19 | const ( 20 | // CAPEVersion1_1_0 defines version v1.1.0. 21 | CAPEVersion1_1_0 = "v1.1.0" 22 | 23 | // CAPEVersion1_2_0 defines version v1.2.0. 24 | CAPEVersion1_2_0 = "v1.2.0" 25 | 26 | // CAPEVersion1_3_0 defines version v1.3.0. 27 | CAPEVersion1_3_0 = "v1.3.0" 28 | 29 | // CAPEVersionLatest defines version latest. 30 | CAPEVersionLatest = "latest" 31 | 32 | // CAPEVersionDefault defines current CAPE version. 33 | CAPEVersionDefault = CAPEVersion1_3_0 34 | ) 35 | -------------------------------------------------------------------------------- /pkg/version/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package version 18 | 19 | import ( 20 | "github.com/Masterminds/semver/v3" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | 23 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 24 | annotationsutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/annotations" 25 | ) 26 | 27 | // GetCAPEVersion returns the CAPE version of the object. 28 | func GetCAPEVersion(o metav1.Object) string { 29 | annotations := o.GetAnnotations() 30 | if annotations == nil { 31 | return "" 32 | } 33 | 34 | return annotations[infrav1.CAPEVersionAnnotation] 35 | } 36 | 37 | // SetCAPEVersion sets the CAPE version for the object. 38 | func SetCAPEVersion(o metav1.Object, version string) { 39 | if !annotationsutil.HasAnnotation(o, infrav1.CAPEVersionAnnotation) { 40 | annotationsutil.AddAnnotations(o, map[string]string{infrav1.CAPEVersionAnnotation: version}) 41 | } 42 | } 43 | 44 | // SetCurrentCAPEVersion sets the latest CAPE version for the object. 45 | func SetCurrentCAPEVersion(o metav1.Object) { 46 | SetCAPEVersion(o, CAPEVersion()) 47 | } 48 | 49 | // IsCompatibleWithPlacementGroup returns whether the current object can use a placement group. 50 | func IsCompatibleWithPlacementGroup(o metav1.Object) bool { 51 | capeVersion := GetCAPEVersion(o) 52 | if (capeVersion == CAPEVersionLatest) || 53 | (IsSemanticVersion(capeVersion) && capeVersion >= CAPEVersion1_2_0) { 54 | return true 55 | } 56 | 57 | return false 58 | } 59 | 60 | // IsSemanticVersion returns whether the version is an valid Semantic Version. 61 | func IsSemanticVersion(version string) bool { 62 | if _, err := semver.NewVersion(version); err != nil { 63 | return false 64 | } 65 | 66 | return true 67 | } 68 | -------------------------------------------------------------------------------- /pkg/version/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package version 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/onsi/gomega" 23 | 24 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 25 | "github.com/smartxworks/cluster-api-provider-elf/test/fake" 26 | ) 27 | 28 | func TestIsCompatibleWithPlacementGroup(t *testing.T) { 29 | g := gomega.NewGomegaWithT(t) 30 | 31 | t.Run("", func(t *testing.T) { 32 | elfMachine := newElfMachineWithoutCAPEVersion() 33 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) 34 | 35 | elfMachine = newElfMachineWithoutCAPEVersion() 36 | SetCAPEVersion(elfMachine, "") 37 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) 38 | 39 | elfMachine = newElfMachineWithoutCAPEVersion() 40 | SetCAPEVersion(elfMachine, "a") 41 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) 42 | 43 | elfMachine = newElfMachineWithoutCAPEVersion() 44 | SetCAPEVersion(elfMachine, CAPEVersion1_1_0) 45 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) 46 | 47 | elfMachine = newElfMachineWithoutCAPEVersion() 48 | SetCAPEVersion(elfMachine, "v1.1.9") 49 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) 50 | 51 | elfMachine = newElfMachineWithoutCAPEVersion() 52 | SetCurrentCAPEVersion(elfMachine) 53 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeTrue()) 54 | 55 | elfMachine = newElfMachineWithoutCAPEVersion() 56 | SetCAPEVersion(elfMachine, CAPEVersion1_2_0+"-alpha.0") 57 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeTrue()) 58 | 59 | elfMachine = newElfMachineWithoutCAPEVersion() 60 | SetCAPEVersion(elfMachine, CAPEVersionLatest) 61 | g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeTrue()) 62 | }) 63 | } 64 | 65 | func newElfMachineWithoutCAPEVersion() *infrav1.ElfMachine { 66 | elfMachine := fake.NewElfMachine(nil) 67 | delete(elfMachine.Annotations, infrav1.CAPEVersionAnnotation) 68 | 69 | return elfMachine 70 | } 71 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package version 18 | 19 | import ( 20 | "fmt" 21 | "runtime" 22 | ) 23 | 24 | var ( 25 | buildVersion string // TAG_NAME var or git branch tag name 26 | gitMajor string // major version, always numeric 27 | gitMinor string // minor version, numeric possibly followed by "+" 28 | gitVersion string // semantic version, derived by build scripts 29 | gitCommit string // sha1 from git, output of $(git rev-parse HEAD) 30 | gitTreeState string // state of git tree, either "clean" or "dirty" 31 | buildDate string // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 32 | ) 33 | 34 | var ( 35 | // The full CAPE version. 36 | capeVersion string 37 | ) 38 | 39 | func init() { 40 | // buildVersion will be set at compiling phase. 41 | // When executing 'go run main.go' in local env, it will be empty. So set it to "latest" same as building against main branch. 42 | if buildVersion == "" { 43 | buildVersion = CAPEVersionLatest 44 | } 45 | 46 | if IsSemanticVersion(buildVersion) { 47 | capeVersion = buildVersion 48 | } else { 49 | capeVersion = CAPEVersionDefault 50 | } 51 | } 52 | 53 | type Info struct { 54 | BuildVersion string `json:"buildVersion,omitempty"` 55 | Major string `json:"major,omitempty"` 56 | Minor string `json:"minor,omitempty"` 57 | GitVersion string `json:"gitVersion,omitempty"` 58 | GitCommit string `json:"gitCommit,omitempty"` 59 | GitTreeState string `json:"gitTreeState,omitempty"` 60 | BuildDate string `json:"buildDate,omitempty"` 61 | GoVersion string `json:"goVersion,omitempty"` 62 | Compiler string `json:"compiler,omitempty"` 63 | Platform string `json:"platform,omitempty"` 64 | } 65 | 66 | func Get() Info { 67 | return Info{ 68 | BuildVersion: buildVersion, 69 | Major: gitMajor, 70 | Minor: gitMinor, 71 | GitVersion: gitVersion, 72 | GitCommit: gitCommit, 73 | GitTreeState: gitTreeState, 74 | BuildDate: buildDate, 75 | GoVersion: runtime.Version(), 76 | Compiler: runtime.Compiler, 77 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 78 | } 79 | } 80 | 81 | // String returns info as a human-friendly version string. 82 | func (info Info) String() string { 83 | return fmt.Sprintf("Version: %s, BuildDate: %s, GitVersion: %s, GitCommit: %s, GoVersion: %s", 84 | info.BuildVersion, info.BuildDate, info.GitVersion, info.GitCommit, info.GoVersion) 85 | } 86 | 87 | // CAPEVersion returns the full CAPE version, e.g. v1.1.0 or v1.1.0-rc.1. 88 | func CAPEVersion() string { 89 | return capeVersion 90 | } 91 | -------------------------------------------------------------------------------- /test/e2e/README.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | 本文主要介绍如何进行 CAPE 测试。 4 | 5 | ## E2E 6 | 7 | 本小节介绍 CAPE 的端到端测试（E2E）。 8 | 9 | ### Requirements 10 | 11 | 运行 E2E 测试的机器需要满足以下条件： 12 | * 可访问的 Tower 服务 13 | * 可以通过 Tower 访问部署在 ELF 集群上的虚拟机 14 | * 安装有 Ginkgo ([download](https://onsi.github.io/ginkgo/#getting-ginkgo)) 15 | * 安装有 Docker ([download](https://www.docker.com/get-started)) 16 | * 安装有 Kind v0.7.0+ ([download](https://kind.sigs.k8s.io)) 17 | 18 | ### Environment variables 19 | 20 | 运行 E2E 测试前需要先设置相关的环境变量： 21 | 22 | | Environment variable | Description | Example | 23 | | --------------------------- | ------------------------------------ | --------------------------------------------------------------------------- | 24 | | `TOWER_SERVER` | Tower 服务器地址 | `127.0.0.1` | 25 | | `TOWER_USERNAME` | Tower 用户名 | `root` | 26 | | `TOWER_PASSWORD` | Tower 用户密码 | `root` | 27 | | `TOWER_AUTH_MODE` | Tower 认证模式 | `LOCAL` | 28 | | `TOWER_SKIP_TLS_VERIFY` | Tower 验证服务器CA证书 | `false` | 29 | | `ELF_CLUSTER` | ELF 集群 ID | `576ad467-d09e-4235-9dec-b615814ddc7e` | 30 | | `VM_TEMPLATE` | 用来创建 Kubernetes 节点的虚拟机模板（模板名或 ID） | `cl7hao0tseso80758osh921f1` | 31 | | `VM_TEMPLATE_UPGRADE_TO` | 用来升级 Kubernetes 节点的虚拟机模板（模板名或 ID） | `cl7hfp6lgx7ts0758xf3oza3c` | 32 | | `CONTROL_PLANE_ENDPOINT_IP` | Kubernetes 集群的 IP 地址 | `127.0.0.1` | 33 | | `ELF_VLAN` | 虚拟网络 ID | `576ad467-d09e-4235-9dec-b615814ddc7e_c8a1e42d-e0f3-4d50-a190-53209a98f157` | 34 | 35 | ### Running the E2E tests 36 | 37 | 执行以下命令运行 E2E 测试： 38 | 39 | ```shell 40 | make test-e2e 41 | ``` 42 | 43 | 以上命令会构建 CAPE 本地镜像并使用该镜像进行 E2E 测试。 44 | -------------------------------------------------------------------------------- /test/e2e/cape_quick_start_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/ginkgo/v2" 23 | capie2e "sigs.k8s.io/cluster-api/test/e2e" 24 | ) 25 | 26 | var _ = Describe("Cluster Creation using Cluster API quick-start test", func() { 27 | capie2e.QuickStartSpec(context.TODO(), func() capie2e.QuickStartSpecInput { 28 | return capie2e.QuickStartSpecInput{ 29 | E2EConfig: e2eConfig, 30 | ClusterctlConfigPath: clusterctlConfigPath, 31 | BootstrapClusterProxy: bootstrapClusterProxy, 32 | ArtifactFolder: artifactFolder, 33 | SkipCleanup: skipCleanup, 34 | } 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /test/e2e/cluster_upgrade_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/ginkgo/v2" 23 | "sigs.k8s.io/cluster-api/test/framework/clusterctl" 24 | ) 25 | 26 | var _ = Describe("When upgrading a workload cluster with a single control plane machine", func() { 27 | ClusterUpgradeSpec(context.TODO(), func() ClusterUpgradeSpecInput { 28 | return ClusterUpgradeSpecInput{ 29 | E2EConfig: e2eConfig, 30 | ClusterctlConfigPath: clusterctlConfigPath, 31 | BootstrapClusterProxy: bootstrapClusterProxy, 32 | ArtifactFolder: artifactFolder, 33 | SkipCleanup: skipCleanup, 34 | ControlPlaneMachineCount: 1, 35 | WorkerMachineCount: 1, 36 | Flavor: clusterctl.DefaultFlavor, 37 | } 38 | }) 39 | }) 40 | 41 | var _ = Describe("When upgrading a workload cluster with a HA control plane", func() { 42 | ClusterUpgradeSpec(context.TODO(), func() ClusterUpgradeSpecInput { 43 | return ClusterUpgradeSpecInput{ 44 | E2EConfig: e2eConfig, 45 | ClusterctlConfigPath: clusterctlConfigPath, 46 | BootstrapClusterProxy: bootstrapClusterProxy, 47 | ArtifactFolder: artifactFolder, 48 | SkipCleanup: skipCleanup, 49 | ControlPlaneMachineCount: 3, 50 | WorkerMachineCount: 2, 51 | Flavor: clusterctl.DefaultFlavor, 52 | } 53 | }) 54 | }) 55 | 56 | var _ = Describe("When upgrading a workload cluster with a HA control plane using scale-in rollout", func() { 57 | ClusterUpgradeSpec(context.TODO(), func() ClusterUpgradeSpecInput { 58 | return ClusterUpgradeSpecInput{ 59 | E2EConfig: e2eConfig, 60 | ClusterctlConfigPath: clusterctlConfigPath, 61 | BootstrapClusterProxy: bootstrapClusterProxy, 62 | ArtifactFolder: artifactFolder, 63 | SkipCleanup: skipCleanup, 64 | ControlPlaneMachineCount: 3, 65 | WorkerMachineCount: 1, 66 | Flavor: "kcp-scale-in", 67 | } 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /test/e2e/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "path/filepath" 23 | 24 | . "github.com/onsi/ginkgo/v2" 25 | corev1 "k8s.io/api/core/v1" 26 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 27 | "sigs.k8s.io/cluster-api/test/framework" 28 | capiutil "sigs.k8s.io/cluster-api/util" 29 | ) 30 | 31 | // Test suite constants for e2e config variables. 32 | const ( 33 | KubernetesVersionUpgradeFrom = "KUBERNETES_VERSION_UPGRADE_FROM" 34 | KubernetesVersionUpgradeTo = "KUBERNETES_VERSION_UPGRADE_TO" 35 | EtcdVersionUpgradeTo = "ETCD_VERSION_UPGRADE_TO" 36 | CoreDNSVersionUpgradeTo = "COREDNS_VERSION_UPGRADE_TO" 37 | MachineTemplateUpgradeTo = "MACHINE_TEMPLATE_UPGRADE_TO" 38 | VMTemplateUpgradeTo = "VM_TEMPLATE_UPGRADE_TO" 39 | VMTemplate = "VM_TEMPLATE" 40 | ElfServerUsername = "ELF_SERVER_USERNAME" 41 | ElfServerPassword = "ELF_SERVER_PASSWORD" //nolint:gosec 42 | ) 43 | 44 | func Byf(format string, a ...interface{}) { 45 | By(fmt.Sprintf(format, a...)) 46 | } 47 | 48 | func Logf(format string, a ...interface{}) { 49 | fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) 50 | } 51 | 52 | func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) { 53 | Byf("Creating a namespace for hosting the %q test spec", specName) 54 | namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ 55 | Creator: clusterProxy.GetClient(), 56 | ClientSet: clusterProxy.GetClientSet(), 57 | Name: fmt.Sprintf("%s-%s", specName, capiutil.RandomString(6)), 58 | LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()), 59 | }) 60 | 61 | return namespace, cancelWatches 62 | } 63 | 64 | func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) { 65 | Byf("Dumping logs from the %q workload cluster", cluster.Name) 66 | 67 | // Dump all the logs from the workload cluster before deleting them. 68 | clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name)) 69 | 70 | Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name) 71 | 72 | // Dump all Cluster API related resources to artifacts before deleting them. 73 | framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ 74 | Lister: clusterProxy.GetClient(), 75 | Namespace: namespace.Name, 76 | LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"), 77 | }) 78 | 79 | if !skipCleanup { 80 | Byf("Deleting cluster %s/%s", cluster.Namespace, cluster.Name) 81 | // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance 82 | // that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait 83 | // instead of DeleteClusterAndWait 84 | framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ 85 | Client: clusterProxy.GetClient(), 86 | Namespace: namespace.Name, 87 | }, intervalsGetter(specName, "wait-delete-cluster")...) 88 | 89 | Byf("Deleting namespace used for hosting the %q test spec", specName) 90 | framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ 91 | Deleter: clusterProxy.GetClient(), 92 | Name: namespace.Name, 93 | }) 94 | } 95 | 96 | cancelWatches() 97 | } 98 | -------------------------------------------------------------------------------- /test/e2e/daemonset_helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | "strings" 22 | 23 | . "github.com/onsi/ginkgo/v2" 24 | . "github.com/onsi/gomega" 25 | appsv1 "k8s.io/api/apps/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "sigs.k8s.io/cluster-api/test/framework" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | ) 30 | 31 | // WaitForKubeProxyUpgradeInput is the input for WaitForKubeProxyUpgrade. 32 | type WaitForKubeProxyUpgradeInput struct { 33 | Getter framework.Getter 34 | KubernetesVersion string 35 | } 36 | 37 | // WaitForKubeProxyUpgrade waits until kube-proxy version matches with the kubernetes version. This is called during KCP upgrade. 38 | func WaitForKubeProxyUpgrade(ctx context.Context, input WaitForKubeProxyUpgradeInput, intervals ...interface{}) { 39 | By("Ensuring kube-proxy has the correct image") 40 | 41 | Eventually(func() (bool, error) { 42 | ds := &appsv1.DaemonSet{} 43 | 44 | if err := input.Getter.Get(ctx, client.ObjectKey{Name: "kube-proxy", Namespace: metav1.NamespaceSystem}, ds); err != nil { 45 | return false, err 46 | } 47 | 48 | if strings.HasSuffix(ds.Spec.Template.Spec.Containers[0].Image, ":"+input.KubernetesVersion) { 49 | return true, nil 50 | } 51 | 52 | return false, nil 53 | }, intervals...).Should(BeTrue()) 54 | } 55 | -------------------------------------------------------------------------------- /test/e2e/data/cape/metadata.yaml: -------------------------------------------------------------------------------- 1 | # maps release series of major.minor to cluster-api contract version 2 | # the contract version may change between minor or major versions, but *not* 3 | # between patch versions. 4 | # 5 | # update this file only when a new major or minor version is released 6 | apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 7 | kind: Metadata 8 | releaseSeries: 9 | - major: 1 10 | minor: 6 11 | contract: v1beta1 12 | - major: 1 13 | minor: 5 14 | contract: v1beta1 15 | - major: 1 16 | minor: 4 17 | contract: v1beta1 18 | - major: 1 19 | minor: 3 20 | contract: v1beta1 21 | - major: 1 22 | minor: 2 23 | contract: v1beta1 24 | - major: 1 25 | minor: 1 26 | contract: v1beta1 27 | - major: 1 28 | minor: 0 29 | contract: v1beta1 30 | - major: 0 31 | minor: 4 32 | contract: v1beta1 33 | - major: 0 34 | minor: 3 35 | contract: v1beta1 36 | - major: 0 37 | minor: 2 38 | contract: v1beta1 39 | - major: 0 40 | minor: 1 41 | contract: v1beta1 42 | -------------------------------------------------------------------------------- /test/e2e/data/capi/metadata.yaml: -------------------------------------------------------------------------------- 1 | # maps release series of major.minor to cluster-api contract version 2 | # the contract version may change between minor or major versions, but *not* 3 | # between patch versions. 4 | # 5 | # update this file only when a new major or minor version is released 6 | apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 7 | kind: Metadata 8 | releaseSeries: 9 | - major: 1 10 | minor: 8 11 | contract: v1beta1 12 | - major: 1 13 | minor: 7 14 | contract: v1beta1 15 | - major: 1 16 | minor: 6 17 | contract: v1beta1 18 | - major: 1 19 | minor: 5 20 | contract: v1beta1 21 | - major: 1 22 | minor: 4 23 | contract: v1beta1 24 | - major: 1 25 | minor: 3 26 | contract: v1beta1 27 | - major: 1 28 | minor: 2 29 | contract: v1beta1 30 | - major: 1 31 | minor: 1 32 | contract: v1beta1 33 | - major: 1 34 | minor: 0 35 | contract: v1beta1 36 | - major: 0 37 | minor: 4 38 | contract: v1alpha4 39 | - major: 0 40 | minor: 3 41 | contract: v1alpha3 42 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/base/cluster-resource-set-label.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cluster.x-k8s.io/v1beta1 2 | kind: Cluster 3 | metadata: 4 | name: '${CLUSTER_NAME}' 5 | namespace: '${NAMESPACE}' 6 | labels: 7 | cni: "${CLUSTER_NAME}-crs-cni" 8 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/base/cluster-resource-set.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: "cni-${CLUSTER_NAME}-crs-cni" 5 | data: ${CNI_RESOURCES} 6 | --- 7 | apiVersion: addons.cluster.x-k8s.io/v1beta1 8 | kind: ClusterResourceSet 9 | metadata: 10 | name: "${CLUSTER_NAME}-crs-cni" 11 | spec: 12 | strategy: ApplyOnce 13 | clusterSelector: 14 | matchLabels: 15 | cni: "${CLUSTER_NAME}-crs-cni" 16 | resources: 17 | - name: "cni-${CLUSTER_NAME}-crs-cni" 18 | kind: ConfigMap 19 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - cluster-template.yaml 5 | - cluster-resource-set.yaml 6 | patchesStrategicMerge: 7 | - cluster-resource-set-label.yaml 8 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-cp-ha/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../base 5 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-kcp-remediation/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../base 5 | - mhc.yaml 6 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-kcp-remediation/mhc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # MachineHealthCheck object with 3 | # - a selector that targets all the machines with label cluster.x-k8s.io/control-plane="" 4 | # - unhealthyConditions triggering remediation after 10s the condition is set 5 | apiVersion: cluster.x-k8s.io/v1beta1 6 | kind: MachineHealthCheck 7 | metadata: 8 | name: "${CLUSTER_NAME}-mhc-0" 9 | spec: 10 | clusterName: "${CLUSTER_NAME}" 11 | maxUnhealthy: 100% 12 | selector: 13 | matchLabels: 14 | cluster.x-k8s.io/control-plane: "" 15 | unhealthyConditions: 16 | - type: e2e.remediation.condition 17 | status: "False" 18 | timeout: 10s 19 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-kcp-scale-in/kcp-surge.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: KubeadmControlPlane 3 | apiVersion: controlplane.cluster.x-k8s.io/v1beta1 4 | metadata: 5 | name: '${CLUSTER_NAME}-control-plane' 6 | namespace: '${NAMESPACE}' 7 | spec: 8 | rolloutStrategy: 9 | rollingUpdate: 10 | maxSurge: 0 11 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-kcp-scale-in/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../base 5 | patchesStrategicMerge: 6 | - kcp-surge.yaml 7 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-md-remediation/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../base 5 | - mhc.yaml 6 | patchesStrategicMerge: 7 | - ./md.yaml 8 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-md-remediation/md.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cluster.x-k8s.io/v1beta1 2 | kind: MachineDeployment 3 | metadata: 4 | name: '${CLUSTER_NAME}-md-0' 5 | namespace: '${NAMESPACE}' 6 | spec: 7 | template: 8 | metadata: 9 | labels: 10 | "e2e.remediation.label": "" 11 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-md-remediation/mhc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # MachineHealthCheck object with 3 | # - a selector that targets all the machines with label e2e.remediation.label="" 4 | # - unhealthyConditions triggering remediation after 10s the condition is set 5 | apiVersion: cluster.x-k8s.io/v1beta1 6 | kind: MachineHealthCheck 7 | metadata: 8 | name: '${CLUSTER_NAME}-mhc-0' 9 | spec: 10 | clusterName: '${CLUSTER_NAME}' 11 | maxUnhealthy: 100% 12 | selector: 13 | matchLabels: 14 | e2e.remediation.label: "" 15 | unhealthyConditions: 16 | - type: e2e.remediation.condition 17 | status: "False" 18 | timeout: 10s 19 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-node-drain/kcp-drain.yaml: -------------------------------------------------------------------------------- 1 | # KubeadmControlPlane referenced by the Cluster object with 2 | # - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test. 3 | kind: KubeadmControlPlane 4 | apiVersion: controlplane.cluster.x-k8s.io/v1beta1 5 | metadata: 6 | name: '${CLUSTER_NAME}-control-plane' 7 | namespace: '${NAMESPACE}' 8 | spec: 9 | machineTemplate: 10 | nodeDrainTimeout: ${NODE_DRAIN_TIMEOUT} 11 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-node-drain/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../base 5 | patchesStrategicMerge: 6 | - kcp-drain.yaml 7 | - md-drain.yaml 8 | # - vcpu.yaml 9 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-node-drain/md-drain.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cluster.x-k8s.io/v1beta1 2 | kind: MachineDeployment 3 | metadata: 4 | name: '${CLUSTER_NAME}-md-0' 5 | namespace: '${NAMESPACE}' 6 | spec: 7 | template: 8 | spec: 9 | nodeDrainTimeout: "${NODE_DRAIN_TIMEOUT}" 10 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/cluster-template-node-drain/vcpu.yaml: -------------------------------------------------------------------------------- 1 | # ElfMachineTemplate object with the number of CPUs raised to 4 2 | # for the purposes of mitigating the CPU spikes caused by scaling up 3 | # the control plane (during upgrades and for HA control planes) 4 | --- 5 | apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 6 | kind: ElfMachineTemplate 7 | metadata: 8 | name: '${CLUSTER_NAME}' 9 | namespace: '${NAMESPACE}' 10 | spec: 11 | template: 12 | spec: 13 | numCPUs: 4 14 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/conformance/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../base 5 | patchesStrategicMerge: 6 | - worker-node-md.yaml 7 | - worker-node-size.yaml 8 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/conformance/worker-node-md.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cluster.x-k8s.io/v1beta1 2 | kind: MachineDeployment 3 | metadata: 4 | name: '${CLUSTER_NAME}-md-0' 5 | namespace: '${NAMESPACE}' 6 | spec: 7 | template: 8 | spec: 9 | infrastructureRef: 10 | name: '${CLUSTER_NAME}-worker' 11 | -------------------------------------------------------------------------------- /test/e2e/data/infrastructure-elf/kustomization/conformance/worker-node-size.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 2 | kind: ElfMachineTemplate 3 | metadata: 4 | name: '${CLUSTER_NAME}-worker' 5 | namespace: '${NAMESPACE}' 6 | spec: 7 | template: 8 | spec: 9 | memoryMiB: 4096 10 | numCPUs: 4 11 | -------------------------------------------------------------------------------- /test/e2e/k8s_conformance_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | // import ( 20 | // "context" 21 | 22 | // . "github.com/onsi/ginkgo" 23 | // capi_e2e "sigs.k8s.io/cluster-api/test/e2e" 24 | // ) 25 | 26 | // var _ = Describe("When testing K8S conformance [Conformance]", func() { 27 | // capi_e2e.K8SConformanceSpec(context.TODO(), func() capi_e2e.K8SConformanceSpecInput { 28 | // return capi_e2e.K8SConformanceSpecInput{ 29 | // E2EConfig: e2eConfig, 30 | // ClusterctlConfigPath: clusterctlConfigPath, 31 | // BootstrapClusterProxy: bootstrapClusterProxy, 32 | // ArtifactFolder: artifactFolder, 33 | // SkipCleanup: skipCleanup, 34 | // Flavor: "conformance", 35 | // } 36 | // }) 37 | // }) 38 | -------------------------------------------------------------------------------- /test/e2e/md_remediations_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/ginkgo/v2" 23 | capie2e "sigs.k8s.io/cluster-api/test/e2e" 24 | ) 25 | 26 | var _ = Describe("Machines remediation using CAPI remediation test", func() { 27 | capie2e.MachineDeploymentRemediationSpec(context.TODO(), func() capie2e.MachineDeploymentRemediationSpecInput { 28 | return capie2e.MachineDeploymentRemediationSpecInput{ 29 | E2EConfig: e2eConfig, 30 | ClusterctlConfigPath: clusterctlConfigPath, 31 | BootstrapClusterProxy: bootstrapClusterProxy, 32 | ArtifactFolder: artifactFolder, 33 | SkipCleanup: skipCleanup, 34 | } 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /test/e2e/md_rollout_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/ginkgo/v2" 23 | capie2e "sigs.k8s.io/cluster-api/test/e2e" 24 | ) 25 | 26 | var _ = Describe("Machines rollout using CAPI rollout test", func() { 27 | capie2e.MachineDeploymentRolloutSpec(context.TODO(), func() capie2e.MachineDeploymentRolloutSpecInput { 28 | return capie2e.MachineDeploymentRolloutSpecInput{ 29 | E2EConfig: e2eConfig, 30 | ClusterctlConfigPath: clusterctlConfigPath, 31 | BootstrapClusterProxy: bootstrapClusterProxy, 32 | ArtifactFolder: artifactFolder, 33 | SkipCleanup: skipCleanup, 34 | } 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /test/e2e/md_scale_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/ginkgo/v2" 23 | capie2e "sigs.k8s.io/cluster-api/test/e2e" 24 | ) 25 | 26 | var _ = Describe("Machines scale using CAPI scale test", func() { 27 | capie2e.MachineDeploymentScaleSpec(context.TODO(), func() capie2e.MachineDeploymentScaleSpecInput { 28 | return capie2e.MachineDeploymentScaleSpecInput{ 29 | E2EConfig: e2eConfig, 30 | ClusterctlConfigPath: clusterctlConfigPath, 31 | BootstrapClusterProxy: bootstrapClusterProxy, 32 | ArtifactFolder: artifactFolder, 33 | SkipCleanup: skipCleanup, 34 | } 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /test/e2e/node_drain_timeout_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/ginkgo/v2" 23 | capie2e "sigs.k8s.io/cluster-api/test/e2e" 24 | ) 25 | 26 | var _ = Describe("When testing node drain timeout", func() { 27 | capie2e.NodeDrainTimeoutSpec(context.TODO(), func() capie2e.NodeDrainTimeoutSpecInput { 28 | return capie2e.NodeDrainTimeoutSpecInput{ 29 | E2EConfig: e2eConfig, 30 | ClusterctlConfigPath: clusterctlConfigPath, 31 | BootstrapClusterProxy: bootstrapClusterProxy, 32 | ArtifactFolder: artifactFolder, 33 | SkipCleanup: skipCleanup, 34 | } 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /test/e2e/node_helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/gomega" 23 | corev1 "k8s.io/api/core/v1" 24 | "sigs.k8s.io/cluster-api/controllers/noderefutil" 25 | "sigs.k8s.io/cluster-api/test/framework" 26 | ) 27 | 28 | // WaitForNodeNotReadyInput is the input for WaitForNodeNotReady. 29 | type WaitForNodeNotReadyInput struct { 30 | Lister framework.Lister 31 | ReadyCount int 32 | NotReadyNodeName string 33 | WaitForNodeNotReady []interface{} 34 | } 35 | 36 | // WaitForNodeNotReady waits until there is a not ready node 37 | // and exactly the given count nodes and they are ready. 38 | func WaitForNodeNotReady(ctx context.Context, input WaitForNodeNotReadyInput) { 39 | Eventually(func() (bool, error) { 40 | nodeList := &corev1.NodeList{} 41 | if err := input.Lister.List(ctx, nodeList); err != nil { 42 | return false, err 43 | } 44 | 45 | nodeReadyCount := 0 46 | notReadyNodeName := "" 47 | for i := range nodeList.Items { 48 | if noderefutil.IsNodeReady(&nodeList.Items[i]) { 49 | nodeReadyCount++ 50 | } else { 51 | notReadyNodeName = nodeList.Items[i].Name 52 | } 53 | } 54 | 55 | return input.ReadyCount == nodeReadyCount && input.NotReadyNodeName == notReadyNodeName, nil 56 | }, input.WaitForNodeNotReady...).Should(BeTrue()) 57 | } 58 | -------------------------------------------------------------------------------- /test/e2e/tower_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | goctx "context" 21 | "flag" 22 | "os" 23 | 24 | env "github.com/caitlinelfring/go-env-default" 25 | . "github.com/onsi/gomega" 26 | ctrllog "sigs.k8s.io/controller-runtime/pkg/log" 27 | 28 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 29 | "github.com/smartxworks/cluster-api-provider-elf/pkg/service" 30 | ) 31 | 32 | var ( 33 | vmTemplate = os.Getenv("VM_TEMPLATE") 34 | vmTemplateUpgradeTo = os.Getenv("VM_TEMPLATE_UPGRADE_TO") 35 | towerUsername = os.Getenv("TOWER_USERNAME") 36 | towerPassword = os.Getenv("TOWER_PASSWORD") 37 | towerAuthMode = os.Getenv("TOWER_AUTH_MODE") 38 | towerSkipTLSVerify = env.GetBoolDefault("TOWER_SKIP_TLS_VERIFY", false) 39 | 40 | towerServer string 41 | vmService service.VMService 42 | ) 43 | 44 | func init() { 45 | flag.StringVar(&towerServer, "e2e.towerServer", os.Getenv("TOWER_SERVER"), "the tower server used for e2e tests") 46 | } 47 | 48 | func initTowerSession() { 49 | var err error 50 | vmService, err = service.NewVMService(goctx.Background(), infrav1.Tower{ 51 | Server: towerServer, 52 | Username: towerUsername, 53 | Password: towerPassword, 54 | AuthMode: towerAuthMode, 55 | SkipTLSVerify: towerSkipTLSVerify}, ctrllog.Log) 56 | Expect(err).ShouldNot(HaveOccurred()) 57 | 58 | template, err := vmService.GetVMTemplate(vmTemplate) 59 | Expect(err).ShouldNot(HaveOccurred()) 60 | Expect(*template.ID).Should(Equal(vmTemplate)) 61 | 62 | template, err = vmService.GetVMTemplate(vmTemplateUpgradeTo) 63 | Expect(err).ShouldNot(HaveOccurred()) 64 | Expect(*template.ID).Should(Equal(vmTemplateUpgradeTo)) 65 | } 66 | -------------------------------------------------------------------------------- /test/e2e/vm_helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "context" 21 | 22 | . "github.com/onsi/gomega" 23 | "github.com/smartxworks/cloudtower-go-sdk/v2/models" 24 | 25 | "github.com/smartxworks/cluster-api-provider-elf/pkg/service" 26 | ) 27 | 28 | // ShutDownVMInput is the input for ShutDownVM. 29 | type ShutDownVMInput struct { 30 | UUID string 31 | VMService service.VMService 32 | WaitVMJobIntervals []interface{} 33 | } 34 | 35 | // ShutDownVM shut down a VM. 36 | func ShutDownVM(ctx context.Context, input ShutDownVMInput) { 37 | task, err := input.VMService.ShutDown(input.UUID) 38 | Expect(err).ShouldNot(HaveOccurred()) 39 | 40 | Eventually(func() (bool, error) { 41 | task, err = input.VMService.GetTask(*task.ID) 42 | if err != nil { 43 | return false, err 44 | } 45 | 46 | if *task.Status == models.TaskStatusSUCCESSED { 47 | return true, nil 48 | } else if *task.Status == models.TaskStatusFAILED { 49 | task, err = input.VMService.ShutDown(input.UUID) 50 | 51 | return false, err 52 | } 53 | 54 | return false, nil 55 | }, input.WaitVMJobIntervals...).Should(BeTrue()) 56 | } 57 | 58 | // PowerOnVMInput is the input for PowerOnVM. 59 | type PowerOnVMInput struct { 60 | UUID string 61 | VMService service.VMService 62 | WaitVMJobIntervals []interface{} 63 | } 64 | 65 | // PowerOnVM power on a VM. 66 | func PowerOnVM(ctx context.Context, input PowerOnVMInput, intervals ...interface{}) { 67 | task, err := input.VMService.PowerOn(input.UUID, "") 68 | Expect(err).ShouldNot(HaveOccurred()) 69 | 70 | Eventually(func() (bool, error) { 71 | task, err = input.VMService.GetTask(*task.ID) 72 | if err != nil { 73 | return false, err 74 | } 75 | 76 | if *task.Status == models.TaskStatusSUCCESSED { 77 | return true, nil 78 | } else if *task.Status == models.TaskStatusFAILED { 79 | task, err = input.VMService.PowerOn(input.UUID, "") 80 | 81 | return false, err 82 | } 83 | 84 | return false, nil 85 | }, input.WaitVMJobIntervals...).Should(BeTrue()) 86 | } 87 | -------------------------------------------------------------------------------- /test/fake/controller_manager_context.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package fake 18 | 19 | import ( 20 | agentv1 "github.com/smartxworks/host-config-agent-api/api/v1alpha1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 23 | cgscheme "k8s.io/client-go/kubernetes/scheme" 24 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 25 | controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 28 | 29 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 30 | "github.com/smartxworks/cluster-api-provider-elf/pkg/context" 31 | ) 32 | 33 | const ( 34 | // ControllerManagerName is the name of the fake controller manager. 35 | ControllerManagerName = "fake-controller-manager" 36 | 37 | // ControllerManagerNamespace is the name of the namespace in which the 38 | // fake controller manager's resources are located. 39 | ControllerManagerNamespace = "fake-cape-system" 40 | 41 | // LeaderElectionNamespace is the namespace used to control leader election 42 | // for the fake controller manager. 43 | LeaderElectionNamespace = ControllerManagerNamespace 44 | 45 | // LeaderElectionID is the name of the ID used to control leader election 46 | // for the fake controller manager. 47 | LeaderElectionID = ControllerManagerName + "-runtime" 48 | ) 49 | 50 | // NewControllerManagerContext returns a fake ControllerManagerContext for unit 51 | // testing reconcilers and webhooks with a fake client. You can choose to 52 | // initialize it with a slice of runtime.Object. 53 | func NewControllerManagerContext(initObjects ...client.Object) *context.ControllerManagerContext { 54 | scheme := runtime.NewScheme() 55 | utilruntime.Must(cgscheme.AddToScheme(scheme)) 56 | utilruntime.Must(clusterv1.AddToScheme(scheme)) 57 | utilruntime.Must(controlplanev1.AddToScheme(scheme)) 58 | utilruntime.Must(infrav1.AddToScheme(scheme)) 59 | utilruntime.Must(agentv1.AddToScheme(scheme)) 60 | 61 | clientWithObjects := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource( 62 | &infrav1.ElfCluster{}, 63 | &infrav1.ElfMachine{}, 64 | ).WithObjects(initObjects...).Build() 65 | 66 | return &context.ControllerManagerContext{ 67 | Client: clientWithObjects, 68 | Scheme: scheme, 69 | Namespace: ControllerManagerNamespace, 70 | Name: ControllerManagerName, 71 | LeaderElectionNamespace: LeaderElectionNamespace, 72 | LeaderElectionID: LeaderElectionID, 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /test/helpers/cluster.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | goctx "context" 5 | "os" 6 | 7 | "github.com/pkg/errors" 8 | corev1 "k8s.io/api/core/v1" 9 | apierrors "k8s.io/apimachinery/pkg/api/errors" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 12 | capisecret "sigs.k8s.io/cluster-api/util/secret" 13 | "sigs.k8s.io/controller-runtime/pkg/client" 14 | ) 15 | 16 | // CreateKubeConfigSecret uses kubeconfig of testEnv to create the workload cluster kubeconfig secret. 17 | func CreateKubeConfigSecret(testEnv *TestEnvironment, namespace, clusterName string) error { 18 | // Return if the secret already exists 19 | if s, err := GetKubeConfigSecret(testEnv, namespace, clusterName); err != nil || s != nil { 20 | return err 21 | } 22 | 23 | secret, err := NewKubeConfigSecret(testEnv, namespace, clusterName) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | return testEnv.CreateAndWait(goctx.Background(), secret) 29 | } 30 | 31 | // GetKubeConfigSecret uses kubeconfig of testEnv to get the workload cluster kubeconfig secret. 32 | func GetKubeConfigSecret(testEnv *TestEnvironment, namespace, clusterName string) (*corev1.Secret, error) { 33 | var secret corev1.Secret 34 | secretKey := client.ObjectKey{ 35 | Namespace: namespace, 36 | Name: capisecret.Name(clusterName, capisecret.Kubeconfig), 37 | } 38 | if err := testEnv.Get(goctx.Background(), secretKey, &secret); err != nil { 39 | if apierrors.IsNotFound(err) { 40 | return nil, nil 41 | } 42 | return nil, errors.Wrapf(err, "failed to get kubeconfig secret %s/%s", secretKey.Namespace, secretKey.Name) 43 | } 44 | return &secret, nil 45 | } 46 | 47 | // DeleteKubeConfigSecret delete the workload cluster kubeconfig secret. 48 | func DeleteKubeConfigSecret(testEnv *TestEnvironment, namespace, clusterName string) error { 49 | deleteSecret := &corev1.Secret{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Namespace: namespace, 52 | Name: capisecret.Name(clusterName, capisecret.Kubeconfig), 53 | }, 54 | } 55 | 56 | if err := testEnv.Delete(goctx.Background(), deleteSecret); err != nil { 57 | if apierrors.IsNotFound(err) { 58 | return nil 59 | } 60 | return errors.Wrapf(err, "failed to delete kubeconfig secret %s/%s", deleteSecret.Namespace, deleteSecret.Name) 61 | } 62 | return nil 63 | } 64 | 65 | // NewKubeConfigSecret uses kubeconfig of testEnv to generate the workload cluster kubeconfig secret. 66 | func NewKubeConfigSecret(testEnv *TestEnvironment, namespace, clusterName string) (*corev1.Secret, error) { 67 | bs, err := os.ReadFile(testEnv.Kubeconfig) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | return &corev1.Secret{ 73 | ObjectMeta: metav1.ObjectMeta{ 74 | Name: capisecret.Name(clusterName, capisecret.Kubeconfig), 75 | Namespace: namespace, 76 | Labels: map[string]string{ 77 | clusterv1.ClusterNameLabel: clusterName, 78 | }, 79 | }, 80 | Data: map[string][]byte{ 81 | capisecret.KubeconfigDataName: bs, 82 | }, 83 | Type: clusterv1.ClusterSecretType, 84 | }, nil 85 | } 86 | -------------------------------------------------------------------------------- /test/helpers/framework.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helpers 18 | 19 | import ( 20 | goctx "context" 21 | "errors" 22 | "fmt" 23 | "os" 24 | "path/filepath" 25 | 26 | "k8s.io/apimachinery/pkg/runtime" 27 | capie2e "sigs.k8s.io/cluster-api/test/e2e" 28 | "sigs.k8s.io/cluster-api/test/framework" 29 | "sigs.k8s.io/cluster-api/test/framework/bootstrap" 30 | "sigs.k8s.io/cluster-api/test/framework/clusterctl" 31 | ) 32 | 33 | // Util functions to interact with the clusterctl e2e framework 34 | 35 | func LoadE2EConfig(configPath string) (*clusterctl.E2EConfig, error) { 36 | config := clusterctl.LoadE2EConfig(goctx.TODO(), clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) 37 | if config == nil { 38 | return nil, fmt.Errorf("cannot load E2E config found at %s", configPath) 39 | } 40 | 41 | return config, nil 42 | } 43 | 44 | func CreateClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string, cniEnabled bool) (string, error) { 45 | createRepositoryInput := clusterctl.CreateRepositoryInput{ 46 | E2EConfig: config, 47 | RepositoryFolder: repositoryFolder, 48 | } 49 | 50 | if cniEnabled { 51 | // Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file as in place of the CNI_RESOURCES envSubst variable. 52 | cniPath, ok := config.Variables[capie2e.CNIPath] 53 | if !ok { 54 | return "", fmt.Errorf("missing %s variable in the config", capie2e.CNIPath) 55 | } 56 | 57 | if _, err := os.Stat(cniPath); err != nil { 58 | return "", fmt.Errorf("the %s variable should resolve to an existing file", capie2e.CNIPath) 59 | } 60 | 61 | createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, capie2e.CNIResources) 62 | } 63 | 64 | clusterctlConfig := clusterctl.CreateRepository(goctx.TODO(), createRepositoryInput) 65 | if _, err := os.Stat(clusterctlConfig); err != nil { 66 | return "", fmt.Errorf("the clusterctl config file does not exists in the local repository %s", repositoryFolder) 67 | } 68 | 69 | return clusterctlConfig, nil 70 | } 71 | 72 | func SetupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy, error) { 73 | var clusterProvider bootstrap.ClusterProvider 74 | kubeconfigPath := "" 75 | if !useExistingCluster { 76 | clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(goctx.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ 77 | Name: config.ManagementClusterName, 78 | RequiresDockerSock: config.HasDockerProvider(), 79 | Images: config.Images, 80 | }) 81 | 82 | kubeconfigPath = clusterProvider.GetKubeconfigPath() 83 | if _, err := os.Stat(kubeconfigPath); err != nil { 84 | return nil, nil, errors.New("failed to get the kubeconfig file for the bootstrap cluster") 85 | } 86 | } 87 | 88 | clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) 89 | 90 | return clusterProvider, clusterProxy, nil 91 | } 92 | 93 | func InitBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config *clusterctl.E2EConfig, clusterctlConfig, artifactFolder string) { 94 | clusterctl.InitManagementClusterAndWatchControllerLogs(goctx.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ 95 | ClusterProxy: bootstrapClusterProxy, 96 | ClusterctlConfigPath: clusterctlConfig, 97 | InfrastructureProviders: config.InfrastructureProviders(), 98 | LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), 99 | }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) 100 | } 101 | 102 | func TearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { 103 | if bootstrapClusterProxy != nil { 104 | bootstrapClusterProxy.Dispose(goctx.TODO()) 105 | } 106 | if bootstrapClusterProvider != nil { 107 | bootstrapClusterProvider.Dispose(goctx.TODO()) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /test/helpers/mod.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helpers 18 | 19 | import ( 20 | "os" 21 | "path/filepath" 22 | 23 | "github.com/pkg/errors" 24 | "golang.org/x/mod/modfile" 25 | ) 26 | 27 | // Mod wraps a go.mod file. 28 | type Mod struct { 29 | path string 30 | content []byte 31 | } 32 | 33 | // NewMod created a new Mod reading a go.mod file. 34 | func NewMod(path string) (Mod, error) { 35 | var mod Mod 36 | content, err := os.ReadFile(filepath.Clean(path)) 37 | if err != nil { 38 | return mod, err 39 | } 40 | return Mod{ 41 | path: path, 42 | content: content, 43 | }, nil 44 | } 45 | 46 | // FindDependencyVersion return the version of a go.mod dependency. 47 | func (m Mod) FindDependencyVersion(dependency string) (string, error) { 48 | f, err := modfile.Parse(m.path, m.content, nil) 49 | if err != nil { 50 | return "", err 51 | } 52 | 53 | var version string 54 | for _, entry := range f.Require { 55 | if entry.Mod.Path == dependency { 56 | version = entry.Mod.Version 57 | break 58 | } 59 | } 60 | if version == "" { 61 | return version, errors.Errorf("could not find required package: %s", dependency) 62 | } 63 | 64 | for _, entry := range f.Replace { 65 | if entry.New.Path == dependency && entry.New.Version != "" { 66 | version = entry.New.Version 67 | } 68 | } 69 | return version, nil 70 | } 71 | -------------------------------------------------------------------------------- /test/helpers/mod_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helpers 18 | 19 | import ( 20 | "os" 21 | "testing" 22 | ) 23 | 24 | func TestMod_FindDependencyVersion(t *testing.T) { 25 | goModData := `module sigs.k8s.io/dummy-project 26 | 27 | go 1.22 28 | 29 | require ( 30 | github.com/foo/bar v1.0.0 31 | github.com/foo/baz v0.9.1 32 | ) 33 | 34 | replace ( 35 | github.com/foo/bar v1.0.0 => github.com/foo/bar v1.0.1 36 | ) 37 | ` 38 | tempPath, err := createTempGoMod(goModData) 39 | if err != nil { 40 | t.Fatal("failed to create test file") 41 | } 42 | defer os.RemoveAll(tempPath) 43 | 44 | m, err := NewMod(tempPath) 45 | if err != nil { 46 | t.Fatalf("failed to init %s", err) 47 | } 48 | 49 | t.Run("find version for existing package with replace", func(t *testing.T) { 50 | name := "github.com/foo/bar" 51 | ver, err := m.FindDependencyVersion(name) 52 | if err != nil { 53 | t.Fatalf("failed to find version for package %s", name) 54 | } 55 | if ver != "v1.0.1" { 56 | t.Fatalf("incorrect version %s", ver) 57 | } 58 | }) 59 | 60 | t.Run("find version for non-existing package", func(t *testing.T) { 61 | name := "sigs.k8s.io/no-such-project" 62 | ver, err := m.FindDependencyVersion(name) 63 | if err == nil || ver != "" { 64 | t.Fatalf("found not existent package %s", name) 65 | } 66 | }) 67 | 68 | t.Run("find version for existing package without replace", func(t *testing.T) { 69 | name := "github.com/foo/baz" 70 | ver, err := m.FindDependencyVersion(name) 71 | if err != nil { 72 | t.Fatalf("failed to find version for package %s", name) 73 | } 74 | if ver != "v0.9.1" { 75 | t.Fatalf("incorrect version %s", ver) 76 | } 77 | }) 78 | } 79 | 80 | func createTempGoMod(data string) (string, error) { 81 | dir, err := os.MkdirTemp("", "parse-mod-") 82 | if err != nil { 83 | return "", err 84 | } 85 | f, err := os.CreateTemp(dir, "test.mod") 86 | if err != nil { 87 | return "", err 88 | } 89 | 90 | if _, err := f.WriteString(data); err != nil { 91 | return "", err 92 | } 93 | defer f.Close() 94 | 95 | return f.Name(), nil 96 | } 97 | -------------------------------------------------------------------------------- /webhooks/elfmachine_webhook_mutation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhooks 18 | 19 | import ( 20 | goctx "context" 21 | "encoding/json" 22 | "net/http" 23 | 24 | "github.com/go-logr/logr" 25 | admissionv1 "k8s.io/api/admission/v1" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/webhook" 29 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 30 | 31 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 32 | annotationsutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/annotations" 33 | "github.com/smartxworks/cluster-api-provider-elf/pkg/version" 34 | ) 35 | 36 | func (m *ElfMachineMutation) SetupWebhookWithManager(mgr ctrl.Manager) error { 37 | if m.decoder == nil { 38 | m.decoder = admission.NewDecoder(mgr.GetScheme()) 39 | } 40 | 41 | hookServer := mgr.GetWebhookServer() 42 | hookServer.Register("/mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine", &webhook.Admission{Handler: m}) 43 | return ctrl.NewWebhookManagedBy(mgr). 44 | For(&infrav1.ElfMachine{}). 45 | Complete() 46 | } 47 | 48 | //+kubebuilder:object:generate=false 49 | //+kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=elfmachines,versions=v1beta1,name=mutation.elfmachine.infrastructure.x-k8s.io,admissionReviewVersions=v1 50 | 51 | type ElfMachineMutation struct { 52 | client.Client 53 | decoder admission.Decoder 54 | logr.Logger 55 | } 56 | 57 | func (m *ElfMachineMutation) Handle(ctx goctx.Context, request admission.Request) admission.Response { 58 | var elfMachine infrav1.ElfMachine 59 | if err := m.decoder.Decode(request, &elfMachine); err != nil { 60 | return admission.Errored(http.StatusBadRequest, err) 61 | } 62 | 63 | if request.Operation == admissionv1.Create { 64 | version.SetCurrentCAPEVersion(&elfMachine) 65 | } 66 | 67 | if elfMachine.Spec.NumCoresPerSocket <= 0 { 68 | // Prefer to set the value to be the same as elfMachineTemplate 69 | elfMachineTemplateName := annotationsutil.GetTemplateClonedFromName(&elfMachine) 70 | if elfMachineTemplateName != "" { 71 | var elfMachineTemplate infrav1.ElfMachineTemplate 72 | if err := m.Get(ctx, client.ObjectKey{ 73 | Namespace: elfMachine.Namespace, 74 | Name: annotationsutil.GetTemplateClonedFromName(&elfMachine), 75 | }, &elfMachineTemplate); err != nil { 76 | return admission.Errored(http.StatusBadRequest, err) 77 | } 78 | elfMachine.Spec.NumCoresPerSocket = elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket 79 | } 80 | // If elfMachineTemplate also has no value, set it to be the same as NumCPUs 81 | if elfMachine.Spec.NumCoresPerSocket <= 0 { 82 | elfMachine.Spec.NumCoresPerSocket = elfMachine.Spec.NumCPUs 83 | } 84 | } 85 | 86 | if marshaledElfMachine, err := json.Marshal(elfMachine); err != nil { 87 | return admission.Errored(http.StatusInternalServerError, err) 88 | } else { 89 | return admission.PatchResponseFromRaw(request.Object.Raw, marshaledElfMachine) 90 | } 91 | } 92 | 93 | // InjectDecoder injects the decoder. 94 | func (m *ElfMachineMutation) InjectDecoder(d admission.Decoder) error { 95 | m.decoder = d 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /webhooks/elfmachine_webhook_mutation_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhooks 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "testing" 23 | 24 | . "github.com/onsi/gomega" 25 | "gomodules.xyz/jsonpatch/v2" 26 | admissionv1 "k8s.io/api/admission/v1" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 30 | "sigs.k8s.io/controller-runtime/pkg/client" 31 | fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" 32 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 33 | 34 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 35 | "github.com/smartxworks/cluster-api-provider-elf/pkg/util/annotations" 36 | "github.com/smartxworks/cluster-api-provider-elf/pkg/version" 37 | "github.com/smartxworks/cluster-api-provider-elf/test/fake" 38 | ) 39 | 40 | func init() { 41 | scheme = runtime.NewScheme() 42 | _ = infrav1.AddToScheme(scheme) 43 | _ = admissionv1.AddToScheme(scheme) 44 | } 45 | 46 | var ( 47 | scheme *runtime.Scheme 48 | ) 49 | 50 | func TestElfMachineMutation(t *testing.T) { 51 | g := NewWithT(t) 52 | scheme := newScheme(g) 53 | tests := []testCase{} 54 | 55 | elfMachine := fake.NewElfMachine(nil) 56 | elfMachine.Annotations = nil 57 | elfMachine.Spec.NumCoresPerSocket = 0 58 | raw, err := marshal(elfMachine) 59 | g.Expect(err).NotTo(HaveOccurred()) 60 | tests = append(tests, testCase{ 61 | name: "should set CAPE version and numCoresPerSocket", 62 | admissionRequest: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ 63 | Kind: metav1.GroupVersionKind{Group: infrav1.GroupVersion.Group, Version: infrav1.GroupVersion.Version, Kind: "ElfMachine"}, 64 | Operation: admissionv1.Create, 65 | Object: runtime.RawExtension{Raw: raw}, 66 | }}, 67 | expectRespAllowed: true, 68 | expectPatchs: []jsonpatch.Operation{ 69 | {Operation: "add", Path: "/metadata/annotations", Value: map[string]interface{}{infrav1.CAPEVersionAnnotation: version.CAPEVersion()}}, 70 | {Operation: "add", Path: "/spec/numCoresPerSocket", Value: float64(elfMachine.Spec.NumCPUs)}, 71 | }, 72 | }) 73 | 74 | elfMachineTemplate := fake.NewElfMachineTemplate() 75 | elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket = 2 76 | elfMachine = fake.NewElfMachine(nil) 77 | annotations.AddAnnotations(elfMachine, map[string]string{clusterv1.TemplateClonedFromNameAnnotation: elfMachineTemplate.Name}) 78 | elfMachine.Spec.NumCoresPerSocket = 0 79 | raw, err = marshal(elfMachine) 80 | g.Expect(err).NotTo(HaveOccurred()) 81 | tests = append(tests, testCase{ 82 | name: "should set NumCoresPerSocket from elfMachineTemplate", 83 | admissionRequest: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ 84 | Kind: metav1.GroupVersionKind{Group: infrav1.GroupVersion.Group, Version: infrav1.GroupVersion.Version, Kind: "ElfMachine"}, 85 | Operation: admissionv1.Update, 86 | Object: runtime.RawExtension{Raw: raw}, 87 | }}, 88 | expectRespAllowed: true, 89 | expectPatchs: []jsonpatch.Operation{ 90 | {Operation: "add", Path: "/spec/numCoresPerSocket", Value: float64(elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket)}, 91 | }, 92 | client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(elfMachineTemplate).Build(), 93 | }) 94 | 95 | for _, tc := range tests { 96 | t.Run(tc.name, func(t *testing.T) { 97 | mutation := ElfMachineMutation{Client: tc.client} 98 | mutation.InjectDecoder(admission.NewDecoder(scheme)) 99 | 100 | resp := mutation.Handle(context.Background(), tc.admissionRequest) 101 | g.Expect(resp.Allowed).Should(Equal(tc.expectRespAllowed)) 102 | g.Expect(resp.Patches).Should(ContainElements(tc.expectPatchs)) 103 | }) 104 | } 105 | } 106 | 107 | func marshal(obj client.Object) ([]byte, error) { 108 | bs, err := json.Marshal(obj) 109 | if err != nil { 110 | return nil, err 111 | } 112 | return bs, nil 113 | } 114 | 115 | type testCase struct { 116 | name string 117 | admissionRequest admission.Request 118 | expectRespAllowed bool 119 | expectPatchs []jsonpatch.Operation 120 | client client.Client 121 | } 122 | -------------------------------------------------------------------------------- /webhooks/elfmachinetemplate_webhook_mutation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhooks 18 | 19 | import ( 20 | goctx "context" 21 | "encoding/json" 22 | "net/http" 23 | 24 | "github.com/go-logr/logr" 25 | "k8s.io/utils/ptr" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/webhook" 29 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 30 | 31 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 32 | ) 33 | 34 | const ( 35 | defaultIPPoolAPIGroup = "ipam.metal3.io" 36 | defaultIPPoolKind = "IPPool" 37 | ) 38 | 39 | func (m *ElfMachineTemplateMutation) SetupWebhookWithManager(mgr ctrl.Manager) error { 40 | if m.decoder == nil { 41 | m.decoder = admission.NewDecoder(mgr.GetScheme()) 42 | } 43 | 44 | hookServer := mgr.GetWebhookServer() 45 | hookServer.Register("/mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachinetemplate", &webhook.Admission{Handler: m}) 46 | return ctrl.NewWebhookManagedBy(mgr). 47 | For(&infrav1.ElfMachine{}). 48 | Complete() 49 | } 50 | 51 | //+kubebuilder:object:generate=false 52 | //+kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachinetemplate,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=elfmachinetemplates,versions=v1beta1,name=mutation.elfmachinetemplate.infrastructure.x-k8s.io,admissionReviewVersions=v1 53 | 54 | type ElfMachineTemplateMutation struct { 55 | client.Client 56 | decoder admission.Decoder 57 | logr.Logger 58 | } 59 | 60 | func (m *ElfMachineTemplateMutation) Handle(ctx goctx.Context, request admission.Request) admission.Response { 61 | var elfMachineTemplate infrav1.ElfMachineTemplate 62 | if err := m.decoder.Decode(request, &elfMachineTemplate); err != nil { 63 | return admission.Errored(http.StatusBadRequest, err) 64 | } 65 | 66 | if elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket <= 0 { 67 | elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket = elfMachineTemplate.Spec.Template.Spec.NumCPUs 68 | } 69 | 70 | devices := elfMachineTemplate.Spec.Template.Spec.Network.Devices 71 | for i := range devices { 72 | for j := range len(devices[i].AddressesFromPools) { 73 | if devices[i].AddressesFromPools[j].APIGroup == nil || *devices[i].AddressesFromPools[j].APIGroup == "" { 74 | devices[i].AddressesFromPools[j].APIGroup = ptr.To(defaultIPPoolAPIGroup) 75 | } 76 | if devices[i].AddressesFromPools[j].Kind == "" { 77 | devices[i].AddressesFromPools[j].Kind = defaultIPPoolKind 78 | } 79 | } 80 | } 81 | 82 | if marshaledElfMachineTemplate, err := json.Marshal(elfMachineTemplate); err != nil { 83 | return admission.Errored(http.StatusInternalServerError, err) 84 | } else { 85 | return admission.PatchResponseFromRaw(request.Object.Raw, marshaledElfMachineTemplate) 86 | } 87 | } 88 | 89 | // InjectDecoder injects the decoder. 90 | func (m *ElfMachineTemplateMutation) InjectDecoder(d admission.Decoder) error { 91 | m.decoder = d 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /webhooks/elfmachinetemplate_webhook_mutation_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhooks 18 | 19 | import ( 20 | "context" 21 | "testing" 22 | 23 | . "github.com/onsi/gomega" 24 | "gomodules.xyz/jsonpatch/v2" 25 | admissionv1 "k8s.io/api/admission/v1" 26 | corev1 "k8s.io/api/core/v1" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | "k8s.io/utils/ptr" 30 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 31 | 32 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 33 | ) 34 | 35 | func TestElfMachineMutationTemplate(t *testing.T) { 36 | g := NewWithT(t) 37 | tests := []testCase{} 38 | 39 | elfMachineTemplate := &infrav1.ElfMachineTemplate{ 40 | ObjectMeta: metav1.ObjectMeta{ 41 | Name: "elfmachinetemplate", 42 | }, 43 | Spec: infrav1.ElfMachineTemplateSpec{ 44 | Template: infrav1.ElfMachineTemplateResource{ 45 | Spec: infrav1.ElfMachineSpec{NumCoresPerSocket: 1}, 46 | }, 47 | }, 48 | } 49 | elfMachineTemplate.Spec.Template.Spec.Network.Devices = []infrav1.NetworkDeviceSpec{ 50 | {AddressesFromPools: []corev1.TypedLocalObjectReference{{Name: "test"}}}, 51 | {AddressesFromPools: []corev1.TypedLocalObjectReference{{Name: "test", APIGroup: ptr.To("")}}}, 52 | {AddressesFromPools: []corev1.TypedLocalObjectReference{{Name: "test", APIGroup: ptr.To("apiGroup")}}}, 53 | {AddressesFromPools: []corev1.TypedLocalObjectReference{{Name: "test", APIGroup: ptr.To("apiGroup"), Kind: "kind"}}}, 54 | } 55 | raw, err := marshal(elfMachineTemplate) 56 | g.Expect(err).NotTo(HaveOccurred()) 57 | tests = append(tests, testCase{ 58 | name: "should set default values for network devices", 59 | admissionRequest: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ 60 | Kind: metav1.GroupVersionKind{Group: infrav1.GroupVersion.Group, Version: infrav1.GroupVersion.Version, Kind: "ElfMachine"}, 61 | Operation: admissionv1.Create, 62 | Object: runtime.RawExtension{Raw: raw}, 63 | }}, 64 | expectRespAllowed: true, 65 | expectPatchs: []jsonpatch.Operation{ 66 | {Operation: "replace", Path: "/spec/template/spec/network/devices/0/addressesFromPools/0/apiGroup", Value: defaultIPPoolAPIGroup}, 67 | {Operation: "replace", Path: "/spec/template/spec/network/devices/0/addressesFromPools/0/kind", Value: defaultIPPoolKind}, 68 | {Operation: "replace", Path: "/spec/template/spec/network/devices/1/addressesFromPools/0/apiGroup", Value: defaultIPPoolAPIGroup}, 69 | {Operation: "replace", Path: "/spec/template/spec/network/devices/1/addressesFromPools/0/kind", Value: defaultIPPoolKind}, 70 | {Operation: "replace", Path: "/spec/template/spec/network/devices/2/addressesFromPools/0/kind", Value: defaultIPPoolKind}, 71 | }, 72 | }) 73 | 74 | elfMachineTemplate.Spec.Template.Spec.Network.Devices = nil 75 | elfMachineTemplate.Spec.Template.Spec.NumCPUs = 1 76 | elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket = 0 77 | raw, err = marshal(elfMachineTemplate) 78 | g.Expect(err).NotTo(HaveOccurred()) 79 | tests = append(tests, testCase{ 80 | name: "should set default values for numCoresPerSocket", 81 | admissionRequest: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ 82 | Kind: metav1.GroupVersionKind{Group: infrav1.GroupVersion.Group, Version: infrav1.GroupVersion.Version, Kind: "ElfMachine"}, 83 | Operation: admissionv1.Create, 84 | Object: runtime.RawExtension{Raw: raw}, 85 | }}, 86 | expectRespAllowed: true, 87 | expectPatchs: []jsonpatch.Operation{ 88 | {Operation: "add", Path: "/spec/template/spec/numCoresPerSocket", Value: float64(elfMachineTemplate.Spec.Template.Spec.NumCPUs)}, 89 | }, 90 | }) 91 | 92 | for _, tc := range tests { 93 | t.Run(tc.name, func(t *testing.T) { 94 | mutation := ElfMachineTemplateMutation{} 95 | mutation.InjectDecoder(admission.NewDecoder(scheme)) 96 | 97 | resp := mutation.Handle(context.Background(), tc.admissionRequest) 98 | g.Expect(resp.Allowed).Should(Equal(tc.expectRespAllowed)) 99 | g.Expect(resp.Patches).Should(HaveLen(len(tc.expectPatchs))) 100 | g.Expect(resp.Patches).Should(ContainElements(tc.expectPatchs)) 101 | }) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /webhooks/elfmachinetemplate_webhook_validation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhooks 18 | 19 | import ( 20 | goctx "context" 21 | "fmt" 22 | 23 | apierrors "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/apimachinery/pkg/util/validation/field" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/webhook" 28 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 29 | 30 | infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" 31 | ) 32 | 33 | // Error messages. 34 | const ( 35 | diskCapacityCannotLessThanZeroMsg = "the disk capacity can only greater than or equal to 0" 36 | memoryCannotLessThanZeroMsg = "the memory can only greater than 0" 37 | numCPUsCannotLessThanZeroMsg = "the umCPUs can only greater than 0" 38 | numCoresPerSocketCannotLessThanZeroMsg = "the numCoresPerSocket can only greater than 0" 39 | ) 40 | 41 | func (v *ElfMachineTemplateValidator) SetupWebhookWithManager(mgr ctrl.Manager) error { 42 | return ctrl.NewWebhookManagedBy(mgr). 43 | For(&infrav1.ElfMachineTemplate{}). 44 | WithValidator(v). 45 | Complete() 46 | } 47 | 48 | //+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachinetemplate,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=elfmachinetemplates,verbs=create;update,versions=v1beta1,name=validation.elfmachinetemplate.infrastructure.x-k8s.io,admissionReviewVersions=v1 49 | 50 | // ElfMachineTemplateValidator implements a validation webhook for ElfMachineTemplate. 51 | type ElfMachineTemplateValidator struct{} 52 | 53 | var _ webhook.CustomValidator = &ElfMachineTemplateValidator{} 54 | 55 | // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. 56 | func (v *ElfMachineTemplateValidator) ValidateCreate(ctx goctx.Context, obj runtime.Object) (admission.Warnings, error) { 57 | elfMachineTemplate, ok := obj.(*infrav1.ElfMachineTemplate) 58 | if !ok { 59 | return nil, apierrors.NewBadRequest(fmt.Sprintf("expected an ElfMachineTemplate but got a %T", obj)) 60 | } 61 | 62 | var allErrs field.ErrorList 63 | if elfMachineTemplate.Spec.Template.Spec.DiskGiB < 0 { 64 | allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "template", "spec", "diskGiB"), elfMachineTemplate.Spec.Template.Spec.DiskGiB, diskCapacityCannotLessThanZeroMsg)) 65 | } 66 | 67 | if elfMachineTemplate.Spec.Template.Spec.MemoryMiB <= 0 { 68 | allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "template", "spec", "memoryMiB"), elfMachineTemplate.Spec.Template.Spec.MemoryMiB, memoryCannotLessThanZeroMsg)) 69 | } 70 | 71 | if elfMachineTemplate.Spec.Template.Spec.NumCPUs <= 0 { 72 | allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "template", "spec", "numCPUs"), elfMachineTemplate.Spec.Template.Spec.NumCPUs, numCPUsCannotLessThanZeroMsg)) 73 | } 74 | 75 | if elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket <= 0 { 76 | allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "template", "spec", "numCoresPerSocket"), elfMachineTemplate.Spec.Template.Spec.NumCoresPerSocket, numCoresPerSocketCannotLessThanZeroMsg)) 77 | } 78 | 79 | return nil, aggregateObjErrors(elfMachineTemplate.GroupVersionKind().GroupKind(), elfMachineTemplate.Name, allErrs) 80 | } 81 | 82 | // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. 83 | func (v *ElfMachineTemplateValidator) ValidateUpdate(ctx goctx.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { 84 | return nil, nil 85 | } 86 | 87 | // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. 88 | func (v *ElfMachineTemplateValidator) ValidateDelete(ctx goctx.Context, obj runtime.Object) (admission.Warnings, error) { 89 | return nil, nil 90 | } 91 | -------------------------------------------------------------------------------- /webhooks/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhooks 18 | 19 | import ( 20 | apierrors "k8s.io/apimachinery/pkg/api/errors" 21 | "k8s.io/apimachinery/pkg/runtime/schema" 22 | "k8s.io/apimachinery/pkg/util/validation/field" 23 | ) 24 | 25 | func aggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error { 26 | if len(allErrs) == 0 { 27 | return nil 28 | } 29 | 30 | return apierrors.NewInvalid( 31 | gk, 32 | name, 33 | allErrs, 34 | ) 35 | } 36 | --------------------------------------------------------------------------------