├── version ├── CHANGES.md ├── .gitignore ├── get_kubernetes_deps.sh ├── NOTICE ├── .github └── workflows │ ├── golangci-lint.yml │ ├── build.yml │ ├── codeql-analysis.yml │ └── build-docker-image.yml ├── Dockerfile ├── nginx-ingress-controller-patch.yml ├── .golangci.yml ├── Makefile ├── traefik-ingress-controller.yml ├── cmd └── cloudstack-ccm │ └── main.go ├── protocol.go ├── deployment.yaml ├── performrelease.sh ├── go.mod ├── protocol_test.go ├── cloudstack_instances_test.go ├── cloudstack_instances.go ├── cloudstack_test.go ├── LICENSE ├── README.md ├── cloudstack.go ├── cloudstack_loadbalancer.go └── go.sum /version: -------------------------------------------------------------------------------- 1 | 1.2.0-snapshot 2 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | Apache CloudStack Kubernetes Provider Changelog 2 | =============================================== 3 | 4 | Version 1.0.0 5 | ------------- 6 | 7 | This is the first release of the CloudStack Kubernetes Provider. 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /cloudstack-ccm 2 | 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Test binary, built with `go test -c` 11 | *.test 12 | 13 | # Code coverage profiles and other test artifacts 14 | *.out 15 | coverage.* 16 | *.coverprofile 17 | profile.cov 18 | 19 | # Dependency directories (remove the comment below to include it) 20 | # vendor/ 21 | 22 | # Go workspace file 23 | go.work 24 | go.work.sum 25 | 26 | # env file 27 | .env 28 | 29 | # Editor/IDE 30 | .idea/ 31 | .vscode/ -------------------------------------------------------------------------------- /get_kubernetes_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | VERSION=${1#"v"} 5 | if [ -z "$VERSION" ]; then 6 | echo "Must specify version!" 7 | exit 1 8 | fi 9 | MODS=($( 10 | curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${VERSION}/go.mod | 11 | sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' 12 | )) 13 | for MOD in "${MODS[@]}"; do 14 | V=$( 15 | go mod download -json "${MOD}@kubernetes-${VERSION}" | 16 | sed -n 's|.*"Version": "\(.*\)".*|\1|p' 17 | ) 18 | go mod edit "-replace=${MOD}=${MOD}@${V}" 19 | done 20 | go get "k8s.io/kubernetes@v${VERSION}" 21 | go mod tidy 22 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache CloudStack Kubernetes Provider 2 | Copyright 2019 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | 8 | The software is based on previous work, which bears the following license: 9 | 10 | Copyright 2016 The Kubernetes Authors. 11 | Copyright 2018 SWISS TXT AG 12 | 13 | Licensed under the Apache License, Version 2.0 (the "License"); 14 | you may not use this file except in compliance with the License. 15 | You may obtain a copy of the License at 16 | 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | 19 | Unless required by applicable law or agreed to in writing, software 20 | distributed under the License is distributed on an "AS IS" BASIS, 21 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 22 | See the License for the specific language governing permissions and 23 | limitations under the License. 24 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: golangci-lint 19 | on: 20 | push: 21 | branches: 22 | - main 23 | pull_request: 24 | 25 | permissions: 26 | contents: read 27 | checks: write 28 | jobs: 29 | golangci: 30 | name: lint 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v6 34 | - uses: actions/setup-go@v6 35 | with: 36 | go-version-file: go.mod 37 | - name: golangci-lint 38 | uses: golangci/golangci-lint-action@v9 39 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: Test-Build 19 | 20 | permissions: 21 | contents: read 22 | 23 | on: [push, pull_request] 24 | 25 | concurrency: 26 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 27 | cancel-in-progress: true 28 | 29 | jobs: 30 | build: 31 | runs-on: ubuntu-latest 32 | 33 | steps: 34 | - uses: actions/checkout@v6 35 | 36 | - name: Set up Go 37 | uses: actions/setup-go@v6 38 | with: 39 | go-version-file: go.mod 40 | 41 | - name: Run Script 42 | run: make test 43 | 44 | - name: Upload coverage to Codecov 45 | uses: codecov/codecov-action@v5 46 | with: 47 | token: ${{ secrets.CODECOV_TOKEN }} 48 | files: coverage.txt 49 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | FROM --platform=$BUILDPLATFORM golang:1.23 AS builder 19 | ARG BUILDPLATFORM 20 | ARG TARGETOS 21 | ARG TARGETARCH 22 | 23 | WORKDIR /go/src/github.com/apache/cloudstack-kubernetes-provider 24 | COPY go.mod /go/src/github.com/apache/cloudstack-kubernetes-provider/go.mod 25 | COPY go.sum /go/src/github.com/apache/cloudstack-kubernetes-provider/go.sum 26 | RUN go mod download 27 | 28 | COPY . /go/src/github.com/apache/cloudstack-kubernetes-provider 29 | RUN make clean && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} make 30 | 31 | FROM gcr.io/distroless/static:nonroot 32 | COPY --from=builder /go/src/github.com/apache/cloudstack-kubernetes-provider/cloudstack-ccm /app/cloudstack-ccm 33 | ENTRYPOINT [ "/app/cloudstack-ccm", "--cloud-provider", "external-cloudstack" ] 34 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: CodeQL Analysis 19 | on: 20 | push: 21 | branches: [main] 22 | pull_request: 23 | branches: [main] 24 | permissions: 25 | actions: read 26 | contents: read 27 | security-events: write 28 | jobs: 29 | codeql: 30 | name: CodeQL 31 | runs-on: ubuntu-latest 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: ["actions", "go"] 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v6 39 | - name: Initialize CodeQL 40 | uses: github/codeql-action/init@v4 41 | with: 42 | languages: ${{ matrix.language }} 43 | - name: Autobuild 44 | uses: github/codeql-action/autobuild@v4 45 | - name: Perform CodeQL Analysis 46 | uses: github/codeql-action/analyze@v4 -------------------------------------------------------------------------------- /nginx-ingress-controller-patch.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | --- 18 | kind: Service 19 | apiVersion: v1 20 | metadata: 21 | name: ingress-nginx 22 | namespace: ingress-nginx 23 | labels: 24 | app.kubernetes.io/name: ingress-nginx 25 | app.kubernetes.io/part-of: ingress-nginx 26 | annotations: 27 | service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol: 'true' 28 | spec: 29 | type: LoadBalancer 30 | selector: 31 | app.kubernetes.io/name: ingress-nginx 32 | app.kubernetes.io/part-of: ingress-nginx 33 | ports: 34 | - name: http 35 | port: 80 36 | targetPort: http 37 | - name: https 38 | port: 443 39 | targetPort: https 40 | --- 41 | kind: ConfigMap 42 | apiVersion: v1 43 | metadata: 44 | name: nginx-configuration 45 | namespace: ingress-nginx 46 | labels: 47 | app.kubernetes.io/name: ingress-nginx 48 | app.kubernetes.io/part-of: ingress-nginx 49 | data: 50 | use-proxy-protocol: "true" 51 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | version: "2" 19 | run: 20 | modules-download-mode: readonly 21 | issues-exit-code: 1 22 | linters: 23 | enable: 24 | - goheader 25 | - gosec 26 | - misspell 27 | - govet 28 | - ineffassign 29 | - staticcheck 30 | - unused 31 | settings: 32 | goheader: 33 | template: |- 34 | * Licensed to the Apache Software Foundation (ASF) under one 35 | * or more contributor license agreements. See the NOTICE file 36 | * distributed with this work for additional information 37 | * regarding copyright ownership. The ASF licenses this file 38 | * to you under the Apache License, Version 2.0 (the 39 | * "License"); you may not use this file except in compliance 40 | * with the License. You may obtain a copy of the License at 41 | * 42 | * http://www.apache.org/licenses/LICENSE-2.0 43 | * 44 | * Unless required by applicable law or agreed to in writing, 45 | * software distributed under the License is distributed on an 46 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 47 | * KIND, either express or implied. See the License for the 48 | * specific language governing permissions and limitations 49 | * under the License. 50 | exclusions: 51 | generated: lax 52 | presets: 53 | - comments 54 | - common-false-positives 55 | - legacy 56 | - std-error-handling 57 | paths: 58 | - third_party$ 59 | - builtin$ 60 | - examples$ 61 | formatters: 62 | exclusions: 63 | generated: lax 64 | paths: 65 | - third_party$ 66 | - builtin$ 67 | - examples$ 68 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') 19 | GIT_COMMIT=$(shell git rev-parse HEAD) 20 | GIT_COMMIT_SHORT=$(shell git rev-parse --short HEAD) 21 | GIT_TAG=$(shell git describe --abbrev=0 --tags 2>/dev/null || echo v0.0.0) 22 | GIT_IS_TAG=$(shell git describe --exact-match --abbrev=0 --tags 2>/dev/null || echo NOT_A_TAG) 23 | ifeq (${GIT_IS_TAG},NOT_A_TAG) 24 | GIT_VERSION?=$(patsubst v%,%,${GIT_TAG})-main+${GIT_COMMIT} 25 | else 26 | GIT_VERSION?=$(patsubst v%,%,${GIT_TAG}) 27 | endif 28 | LDFLAGS="-X k8s.io/kubernetes/pkg/version.gitVersion=${GIT_VERSION} -X k8s.io/kubernetes/pkg/version.gitCommit=${GIT_COMMIT} -X k8s.io/kubernetes/pkg/version.buildDate=${BUILD_DATE}" 29 | export CGO_ENABLED=0 30 | export GO111MODULE=on 31 | 32 | CMD_SRC=\ 33 | cmd/cloudstack-ccm/main.go 34 | 35 | .PHONY: all clean docker 36 | 37 | all: cloudstack-ccm 38 | 39 | clean: 40 | rm -f cloudstack-ccm 41 | 42 | cloudstack-ccm: ${CMD_SRC} 43 | go build -ldflags ${LDFLAGS} -o $@ $^ 44 | 45 | test: gofmt 46 | go test -v -coverprofile=coverage.txt -covermode=atomic 47 | go vet 48 | 49 | docker: gofmt 50 | docker build . -t apache/cloudstack-kubernetes-provider:${GIT_COMMIT_SHORT} 51 | docker tag apache/cloudstack-kubernetes-provider:${GIT_COMMIT_SHORT} apache/cloudstack-kubernetes-provider:latest 52 | ifneq (${GIT_IS_TAG},NOT_A_TAG) 53 | docker tag apache/cloudstack-kubernetes-provider:${GIT_COMMIT_SHORT} apache/cloudstack-kubernetes-provider:${GIT_TAG} 54 | endif 55 | 56 | lint: gofmt 57 | @(echo "Running golangci-lint...") 58 | golangci-lint run 59 | 60 | gofmt: 61 | @(echo "Running gofmt...") 62 | @(echo "gofmt -l"; FMTFILES="$$(gofmt -l .)"; if test -n "$${FMTFILES}"; then echo "Go files that need to be reformatted (use 'go fmt'):\n$${FMTFILES}"; exit 1; fi) 63 | -------------------------------------------------------------------------------- /traefik-ingress-controller.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: traefik 22 | annotations: 23 | service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol: 'true' 24 | spec: 25 | type: LoadBalancer 26 | ports: 27 | - name: http 28 | port: 80 29 | targetPort: http 30 | - name: https 31 | port: 443 32 | targetPort: https 33 | --- 34 | apiVersion: v1 35 | kind: ConfigMap 36 | metadata: 37 | name: traefik-conf 38 | data: 39 | traefik.toml: | 40 | defaultEntryPoints = ["http"] 41 | [entryPoints] 42 | [entryPoints.http] 43 | address = ":80" 44 | [entryPoints.http.proxyProtocol] 45 | trustedIPs = ["127.0.0.1/32", "10.0.0.1/32"] 46 | [entryPoints.https] 47 | address = ":443" 48 | [entryPoints.https.proxyProtocol] 49 | trustedIPs = ["127.0.0.1/32", "10.0.0.1/32"] 50 | --- 51 | apiVersion: apps/v1 52 | kind: DaemonSet 53 | metadata: 54 | name: traefik-ingress-controller 55 | spec: 56 | selector: 57 | matchLabels: 58 | name: traefik-ingress-controller 59 | template: 60 | metadata: 61 | labels: 62 | name: traefik-ingress-controller 63 | spec: 64 | hostNetwork: true 65 | containers: 66 | - args: 67 | - --configfile=/config/traefik.toml 68 | image: traefik:1.7.12 69 | imagePullPolicy: Always 70 | name: traefik-ingress 71 | ports: 72 | - containerPort: 80 73 | hostPort: 80 74 | name: http 75 | protocol: TCP 76 | - containerPort: 443 77 | hostPort: 443 78 | name: https 79 | protocol: TCP 80 | volumeMounts: 81 | - mountPath: /config 82 | name: config 83 | volumes: 84 | - configMap: 85 | defaultMode: 420 86 | name: traefik-conf 87 | name: config 88 | -------------------------------------------------------------------------------- /cmd/cloudstack-ccm/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | goflag "flag" 24 | 25 | "k8s.io/apimachinery/pkg/util/wait" 26 | cloudprovider "k8s.io/cloud-provider" 27 | 28 | "k8s.io/cloud-provider/app" 29 | "k8s.io/cloud-provider/app/config" 30 | "k8s.io/cloud-provider/options" 31 | cliflag "k8s.io/component-base/cli/flag" 32 | "k8s.io/component-base/logs" 33 | 34 | _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins 35 | _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration 36 | "k8s.io/klog/v2" 37 | 38 | "github.com/spf13/pflag" 39 | 40 | _ "github.com/apache/cloudstack-kubernetes-provider" // our cloud package 41 | ) 42 | 43 | func main() { 44 | 45 | ccmOptions, err := options.NewCloudControllerManagerOptions() 46 | if err != nil { 47 | klog.Fatalf("unable to initialize command options: %v", err) 48 | } 49 | 50 | fss := cliflag.NamedFlagSets{} 51 | 52 | command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop) 53 | 54 | // TODO: once we switch everything over to Cobra commands, we can go back to calling 55 | // cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the 56 | // normalize func and add the go flag set by hand. 57 | pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) 58 | pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) 59 | //cliflag.InitFlags() 60 | logs.InitLogs() 61 | defer logs.FlushLogs() 62 | 63 | if err := command.Execute(); err != nil { 64 | klog.Fatalf("error: %v\n", err) 65 | } 66 | } 67 | 68 | func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface { 69 | cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider 70 | 71 | // initialize cloud provider with the cloud provider name and config file provided 72 | cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) 73 | if err != nil { 74 | klog.Fatalf("Cloud provider could not be initialized: %v", err) 75 | } 76 | if cloud == nil { 77 | klog.Fatalf("Cloud provider is nil") 78 | } 79 | 80 | if !cloud.HasClusterID() { 81 | if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { 82 | klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") 83 | } else { 84 | klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") 85 | } 86 | } 87 | return cloud 88 | } 89 | -------------------------------------------------------------------------------- /.github/workflows/build-docker-image.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: Docker Image Build 19 | 20 | on: 21 | push: 22 | branches: 23 | - main 24 | tags: 25 | - '*' 26 | pull_request: 27 | 28 | permissions: 29 | contents: read 30 | 31 | concurrency: 32 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 33 | cancel-in-progress: true 34 | 35 | jobs: 36 | build: 37 | if: github.repository == 'apache/cloudstack-kubernetes-provider' 38 | runs-on: ubuntu-latest 39 | steps: 40 | - name: Set Docker repository name 41 | run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV 42 | 43 | - uses: actions/checkout@v6 44 | with: 45 | fetch-depth: 0 46 | 47 | - name: Set Docker image TAG 48 | run: echo "TAG=$(if [ "${{ github.event_name }}" = "pull_request" ];then echo "pr${{ github.event.pull_request.number}}"; elif [ "${{ github.ref_name }}" = "main" ];then cat version; else echo ${{ github.ref_name }};fi)" >> $GITHUB_ENV 49 | 50 | - name: Set Docker image FULL TAG 51 | run: echo "FULL_TAG=$(if [ "${{ secrets.DOCKER_REGISTRY }}" = "" ];then echo ${DOCKER_REPOSITORY}/cloudstack-kubernetes-provider:${TAG};else echo ${{ secrets.DOCKER_REGISTRY }}/${DOCKER_REPOSITORY}/cloudstack-kubernetes-provider:${TAG};fi)" >> $GITHUB_ENV 52 | 53 | - name: Check if should push 54 | id: should_push 55 | run: | 56 | if [ "${{ github.event_name }}" != "pull_request" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" = "${{ github.repository }}" ]; then 57 | echo "should_push=true" >> $GITHUB_OUTPUT 58 | else 59 | echo "should_push=false" >> $GITHUB_OUTPUT 60 | fi 61 | 62 | - name: Set up Docker Buildx 63 | uses: docker/setup-buildx-action@v3 64 | 65 | - name: Login to Docker Registry 66 | if: steps.should_push.outputs.should_push == 'true' 67 | uses: docker/login-action@v3 68 | with: 69 | registry: ${{ secrets.DOCKER_REGISTRY }} 70 | username: ${{ secrets.DOCKERHUB_USER }} 71 | password: ${{ secrets.DOCKERHUB_TOKEN }} 72 | 73 | - name: Set cache configuration 74 | id: cache_config 75 | run: | 76 | if [ "${{ steps.should_push.outputs.should_push }}" = "true" ]; then 77 | echo "cache_from=type=registry,ref=${FULL_TAG}-cache" >> $GITHUB_OUTPUT 78 | echo "cache_to=type=registry,ref=${FULL_TAG}-cache,mode=max" >> $GITHUB_OUTPUT 79 | fi 80 | 81 | - name: Build and push Docker image for cloudstack-kubernetes-provider (multi-arch) 82 | uses: docker/build-push-action@v6 83 | with: 84 | context: . 85 | file: ./Dockerfile 86 | platforms: linux/amd64,linux/arm64 87 | push: ${{ steps.should_push.outputs.should_push == 'true' }} 88 | tags: ${{ env.FULL_TAG }} 89 | cache-from: ${{ steps.cache_config.outputs.cache_from }} 90 | cache-to: ${{ steps.cache_config.outputs.cache_to }} 91 | -------------------------------------------------------------------------------- /protocol.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | v1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | // LoadBalancerProtocol represents a specific network protocol supported by the CloudStack load balancer. 27 | // 28 | // It also allows easy mapping to standard protocol names. 29 | type LoadBalancerProtocol int 30 | 31 | const ( 32 | LoadBalancerProtocolTCP LoadBalancerProtocol = iota 33 | LoadBalancerProtocolUDP 34 | LoadBalancerProtocolTCPProxy 35 | LoadBalancerProtocolInvalid 36 | ) 37 | 38 | // String returns the same value as CSProtocol. 39 | func (p LoadBalancerProtocol) String() string { 40 | return p.CSProtocol() 41 | } 42 | 43 | // CSProtocol returns the full CloudStack protocol name. 44 | // Returns "" if the value is unknown. 45 | func (p LoadBalancerProtocol) CSProtocol() string { 46 | switch p { 47 | case LoadBalancerProtocolTCP: 48 | return "tcp" 49 | case LoadBalancerProtocolUDP: 50 | return "udp" 51 | case LoadBalancerProtocolTCPProxy: 52 | return "tcp-proxy" 53 | default: 54 | return "" 55 | } 56 | } 57 | 58 | // IPProtocol returns the standard IP protocol name. 59 | // Returns "" if the value is unknown. 60 | func (p LoadBalancerProtocol) IPProtocol() string { 61 | switch p { 62 | case LoadBalancerProtocolTCP: 63 | fallthrough 64 | case LoadBalancerProtocolTCPProxy: 65 | return "tcp" 66 | case LoadBalancerProtocolUDP: 67 | return "udp" 68 | default: 69 | return "" 70 | } 71 | } 72 | 73 | // ProtocolFromServicePort selects a suitable CloudStack protocol type 74 | // based on a ServicePort object and annotations from a LoadBalancer definition. 75 | // 76 | // Supported combinations include: 77 | // 78 | // v1.ProtocolTCP="tcp" -> "tcp" 79 | // v1.ProtocolTCP="udp" -> "udp" (CloudStack 4.6 and later) 80 | // v1.ProtocolTCP="tcp" + annotation "service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol" 81 | // -> "tcp-proxy" (CloudStack 4.6 and later) 82 | // 83 | // Other values return LoadBalancerProtocolInvalid. 84 | func ProtocolFromServicePort(port v1.ServicePort, service *v1.Service) LoadBalancerProtocol { 85 | proxy := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyProtocol, false) 86 | switch port.Protocol { 87 | case v1.ProtocolTCP: 88 | if proxy { 89 | return LoadBalancerProtocolTCPProxy 90 | } else { 91 | return LoadBalancerProtocolTCP 92 | } 93 | case v1.ProtocolUDP: 94 | return LoadBalancerProtocolUDP 95 | default: 96 | return LoadBalancerProtocolInvalid 97 | } 98 | } 99 | 100 | // ProtocolFromLoadBalancer returns the protocol corresponding to the 101 | // CloudStack load balancer protocol name. 102 | func ProtocolFromLoadBalancer(protocol string) LoadBalancerProtocol { 103 | switch protocol { 104 | case "tcp": 105 | return LoadBalancerProtocolTCP 106 | case "udp": 107 | return LoadBalancerProtocolUDP 108 | case "tcp-proxy": 109 | return LoadBalancerProtocolTCPProxy 110 | default: 111 | return LoadBalancerProtocolInvalid 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: cloud-controller-manager 6 | namespace: kube-system 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: ClusterRole 10 | metadata: 11 | name: system:cloud-controller-manager 12 | annotations: 13 | rbac.authorization.kubernetes.io/autoupdate: "true" 14 | labels: 15 | k8s-app: cloud-controller-manager 16 | rules: 17 | - apiGroups: 18 | - coordination.k8s.io 19 | resources: 20 | - leases 21 | verbs: 22 | - get 23 | - create 24 | - update 25 | - apiGroups: 26 | - "" 27 | resources: 28 | - configmaps 29 | verbs: 30 | - list 31 | - watch 32 | - apiGroups: 33 | - "" 34 | resources: 35 | - events 36 | verbs: 37 | - create 38 | - patch 39 | - update 40 | - apiGroups: 41 | - "" 42 | resources: 43 | - nodes 44 | verbs: 45 | - '*' 46 | - apiGroups: 47 | - "" 48 | resources: 49 | - pods 50 | verbs: 51 | - list 52 | - get 53 | - apiGroups: 54 | - "" 55 | resources: 56 | - nodes/status 57 | verbs: 58 | - patch 59 | - apiGroups: 60 | - "" 61 | resources: 62 | - services 63 | verbs: 64 | - get 65 | - list 66 | - patch 67 | - update 68 | - watch 69 | - apiGroups: 70 | - "" 71 | resources: 72 | - services/status 73 | verbs: 74 | - list 75 | - patch 76 | - update 77 | - watch 78 | - apiGroups: 79 | - "" 80 | resources: 81 | - serviceaccounts 82 | verbs: 83 | - create 84 | - apiGroups: 85 | - "" 86 | resources: 87 | - endpoints 88 | verbs: 89 | - create 90 | - get 91 | - list 92 | - watch 93 | - update 94 | - apiGroups: 95 | - "" 96 | resources: 97 | - persistentvolumes 98 | verbs: 99 | - list 100 | - watch 101 | - patch 102 | --- 103 | kind: ClusterRoleBinding 104 | apiVersion: rbac.authorization.k8s.io/v1 105 | metadata: 106 | name: system:cloud-controller-manager 107 | roleRef: 108 | apiGroup: rbac.authorization.k8s.io 109 | kind: ClusterRole 110 | name: system:cloud-controller-manager 111 | subjects: 112 | - kind: ServiceAccount 113 | name: cloud-controller-manager 114 | namespace: kube-system 115 | --- 116 | apiVersion: rbac.authorization.k8s.io/v1 117 | kind: RoleBinding 118 | metadata: 119 | name: system:cloud-controller-manager:extension-apiserver-authentication-reader 120 | namespace: kube-system 121 | roleRef: 122 | apiGroup: rbac.authorization.k8s.io 123 | kind: Role 124 | name: extension-apiserver-authentication-reader 125 | subjects: 126 | - kind: ServiceAccount 127 | name: cloud-controller-manager 128 | namespace: kube-system 129 | --- 130 | apiVersion: apps/v1 131 | kind: Deployment 132 | metadata: 133 | labels: 134 | k8s-app: cloud-controller-manager 135 | name: cloud-controller-manager 136 | namespace: kube-system 137 | spec: 138 | replicas: 1 139 | selector: 140 | matchLabels: 141 | k8s-app: cloud-controller-manager 142 | strategy: 143 | rollingUpdate: 144 | maxSurge: 25% 145 | maxUnavailable: 25% 146 | type: RollingUpdate 147 | template: 148 | metadata: 149 | labels: 150 | k8s-app: cloud-controller-manager 151 | spec: 152 | containers: 153 | - name: cloud-controller-manager 154 | image: apache/cloudstack-kubernetes-provider:v1.2.0 155 | imagePullPolicy: IfNotPresent 156 | args: 157 | - --leader-elect=true 158 | - --cloud-provider=external-cloudstack 159 | - --cloud-config=/config/cloud-config 160 | resources: 161 | limits: 162 | cpu: 50m 163 | memory: 120Mi 164 | requests: 165 | cpu: 10m 166 | memory: 60Mi 167 | volumeMounts: 168 | - name: config-volume 169 | mountPath: /config 170 | restartPolicy: Always 171 | serviceAccountName: cloud-controller-manager 172 | terminationGracePeriodSeconds: 30 173 | volumes: 174 | - name: config-volume 175 | secret: 176 | secretName: cloudstack-secret 177 | tolerations: 178 | # this is required so CCM can bootstrap itself 179 | - key: node.cloudprovider.kubernetes.io/uninitialized 180 | value: "true" 181 | effect: NoSchedule 182 | -------------------------------------------------------------------------------- /performrelease.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | version='TESTBUILD' 20 | sourcedir=~/cloudstack-kubernetes-provider 21 | outputdir=/tmp/cloudstack-kubernetes-provider-build/ 22 | branch='main' 23 | tag='no' 24 | certid='X' 25 | committosvn='X' 26 | 27 | usage(){ 28 | echo "usage: $0 -v version [-b branch] [-s source dir] [-o output dir] [-t] [-u] [-c] [-h]" 29 | echo " -v sets the version" 30 | echo " -b sets the branch (defaults to 'main')" 31 | echo " -s sets the source directory (defaults to $sourcedir)" 32 | echo " -o sets the output directory (defaults to $outputdir)" 33 | echo " -t tags the git repo with the version" 34 | echo " -u sets the certificate ID to sign with (if not provided, the default key is attempted)" 35 | echo " -c commits build artifacts to cloudstack dev dist dir in svn" 36 | echo " -h" 37 | } 38 | 39 | while getopts v:s:o:b:u:tch opt 40 | do 41 | case "$opt" in 42 | v) version="$OPTARG";; 43 | s) sourcedir="$OPTARG";; 44 | o) outputdir="$OPTARG";; 45 | b) branch="$OPTARG";; 46 | t) tag="yes";; 47 | u) certid="$OPTARG";; 48 | c) committosvn="yes";; 49 | h) usage 50 | exit 0;; 51 | /?) # unknown flag 52 | usage 53 | exit 1;; 54 | esac 55 | done 56 | shift `expr $OPTIND - 1` 57 | 58 | if [ $version == "TESTBUILD" ]; then 59 | echo >&2 "A version must be specified with the -v option: performrelease.sh -v 1.0.0" 60 | exit 1 61 | fi 62 | 63 | echo "Using version: $version" 64 | echo "Using source directory: $sourcedir" 65 | echo "Using output directory: $outputdir" 66 | echo "Using branch: $branch" 67 | if [ "$tag" == "yes" ]; then 68 | if [ "$certid" == "X" ]; then 69 | echo "Tagging the branch with the version number, and signing the branch with your default certificate." 70 | else 71 | echo "Tagging the branch with the version number, and signing the branch with certificate ID $certid." 72 | fi 73 | else 74 | echo "The branch will not be tagged. You should consider doing this." 75 | fi 76 | 77 | if [ -d "$outputdir" ]; then 78 | rm -r $outputdir/* 79 | else 80 | mkdir $outputdir 81 | fi 82 | 83 | cd $sourcedir 84 | 85 | echo 'checking out correct branch' 86 | git checkout $branch 87 | 88 | git clean -f 89 | 90 | export commitsh=`git show HEAD | head -n 1 | cut -d ' ' -f 2` 91 | echo "releasing as $commitsh" 92 | 93 | echo 'archiving' 94 | git archive --format=tar --prefix=apache-cloudstack-kubernetes-provider-$version-src/ $branch > $outputdir/apache-cloudstack-kubernetes-provider-$version-src.tar 95 | bzip2 $outputdir/apache-cloudstack-kubernetes-provider-$version-src.tar 96 | 97 | cd $outputdir 98 | echo 'armor' 99 | if [ "$certid" == "X" ]; then 100 | gpg -v --armor --output apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.asc --detach-sig apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 101 | else 102 | gpg -v --default-key $certid --armor --output apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.asc --detach-sig apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 103 | fi 104 | 105 | echo 'md5' 106 | gpg -v --print-md MD5 apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 > apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.md5 107 | 108 | echo 'sha512' 109 | gpg -v --print-md SHA512 apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 > apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.sha512 110 | 111 | echo 'verify' 112 | gpg -v --verify apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.asc apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 113 | 114 | if [ "$tag" == "yes" ]; then 115 | echo 'tag' 116 | cd $sourcedir 117 | if [ "$certid" == "X" ]; then 118 | git tag -s $version -m "Tagging release $version on branch $branch." 119 | else 120 | git tag -u $certid -s $version -m "Tagging release $version on branch $branch." 121 | fi 122 | fi 123 | 124 | if [ "$committosvn" == "yes" ]; then 125 | echo 'committing artifacts to svn' 126 | rm -Rf /tmp/cloudstack-dev-dist 127 | cd /tmp 128 | svn co https://dist.apache.org/repos/dist/dev/cloudstack/ cloudstack-dev-dist 129 | cd cloudstack-dev-dist 130 | if [ -d "kubernetes-provider-$version" ]; then 131 | cd kubernetes-provider-$version 132 | svn rm * 133 | else 134 | mkdir kubernetes-provider-$version 135 | svn add kubernetes-provider-$version 136 | cd kubernetes-provider-$version 137 | fi 138 | cp $outputdir/apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 . 139 | cp $outputdir/apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.asc . 140 | cp $outputdir/apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.md5 . 141 | cp $outputdir/apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.sha512 . 142 | svn add apache-cloudstack-kubernetes-provider-$version-src.tar.bz2 143 | svn add apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.asc 144 | svn add apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.md5 145 | svn add apache-cloudstack-kubernetes-provider-$version-src.tar.bz2.sha512 146 | svn commit -m "Committing release candidate artifacts for $version to dist/dev/cloudstack in preparation for release vote" 147 | fi 148 | 149 | echo "completed. use commit-sh of $commitsh when starting the VOTE thread" 150 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/apache/cloudstack-kubernetes-provider 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/apache/cloudstack-go/v2 v2.19.0 7 | github.com/blang/semver/v4 v4.0.0 8 | github.com/spf13/pflag v1.0.5 9 | go.uber.org/mock v0.5.0 10 | gopkg.in/gcfg.v1 v1.2.3 11 | k8s.io/api v0.24.17 12 | k8s.io/apimachinery v0.24.17 13 | k8s.io/cloud-provider v0.24.17 14 | k8s.io/component-base v0.24.17 15 | k8s.io/klog/v2 v2.80.1 16 | ) 17 | 18 | require ( 19 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 20 | github.com/NYTimes/gziphandler v1.1.1 // indirect 21 | github.com/beorn7/perks v1.0.1 // indirect 22 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 23 | github.com/coreos/go-semver v0.3.0 // indirect 24 | github.com/coreos/go-systemd/v22 v22.3.2 // indirect 25 | github.com/davecgh/go-spew v1.1.1 // indirect 26 | github.com/emicklei/go-restful v2.16.0+incompatible // indirect 27 | github.com/evanphx/json-patch v4.12.0+incompatible // indirect 28 | github.com/felixge/httpsnoop v1.0.4 // indirect 29 | github.com/fsnotify/fsnotify v1.6.0 // indirect 30 | github.com/go-logr/logr v1.4.1 // indirect 31 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 32 | github.com/go-openapi/jsonreference v0.20.0 // indirect 33 | github.com/go-openapi/swag v0.19.14 // indirect 34 | github.com/gogo/protobuf v1.3.2 // indirect 35 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 36 | github.com/golang/protobuf v1.5.4 // indirect 37 | github.com/google/gnostic v0.5.7-v3refs // indirect 38 | github.com/google/go-cmp v0.6.0 // indirect 39 | github.com/google/gofuzz v1.1.0 // indirect 40 | github.com/google/uuid v1.6.0 // indirect 41 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 42 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect 43 | github.com/imdario/mergo v0.3.6 // indirect 44 | github.com/inconshreveable/mousetrap v1.0.1 // indirect 45 | github.com/josharian/intern v1.0.0 // indirect 46 | github.com/json-iterator/go v1.1.12 // indirect 47 | github.com/kr/pretty v0.3.1 // indirect 48 | github.com/mailru/easyjson v0.7.6 // indirect 49 | github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect 50 | github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect 51 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 52 | github.com/modern-go/reflect2 v1.0.2 // indirect 53 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 54 | github.com/pkg/errors v0.9.1 // indirect 55 | github.com/prometheus/client_golang v1.14.0 // indirect 56 | github.com/prometheus/client_model v0.3.0 // indirect 57 | github.com/prometheus/common v0.37.0 // indirect 58 | github.com/prometheus/procfs v0.8.0 // indirect 59 | github.com/spf13/cobra v1.6.0 // indirect 60 | go.etcd.io/etcd/api/v3 v3.5.14 // indirect 61 | go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect 62 | go.etcd.io/etcd/client/v3 v3.5.14 // indirect 63 | go.opentelemetry.io/contrib v0.20.0 // indirect 64 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect 65 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect 66 | go.opentelemetry.io/otel v0.20.0 // indirect 67 | go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect 68 | go.opentelemetry.io/otel/metric v0.20.0 // indirect 69 | go.opentelemetry.io/otel/sdk v0.20.0 // indirect 70 | go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect 71 | go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect 72 | go.opentelemetry.io/otel/trace v0.20.0 // indirect 73 | go.opentelemetry.io/proto/otlp v0.7.0 // indirect 74 | go.uber.org/atomic v1.7.0 // indirect 75 | go.uber.org/multierr v1.6.0 // indirect 76 | go.uber.org/zap v1.19.0 // indirect 77 | golang.org/x/crypto v0.36.0 // indirect 78 | golang.org/x/net v0.38.0 // indirect 79 | golang.org/x/oauth2 v0.18.0 // indirect 80 | golang.org/x/sync v0.12.0 // indirect 81 | golang.org/x/sys v0.31.0 // indirect 82 | golang.org/x/term v0.30.0 // indirect 83 | golang.org/x/text v0.23.0 // indirect 84 | golang.org/x/time v0.3.0 // indirect 85 | google.golang.org/appengine v1.6.8 // indirect 86 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect 87 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect 88 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect 89 | google.golang.org/grpc v1.64.0 // indirect 90 | google.golang.org/protobuf v1.34.1 // indirect 91 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 92 | gopkg.in/inf.v0 v0.9.1 // indirect 93 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 94 | gopkg.in/warnings.v0 v0.1.1 // indirect 95 | gopkg.in/yaml.v2 v2.4.0 // indirect 96 | gopkg.in/yaml.v3 v3.0.1 // indirect 97 | k8s.io/apiserver v0.24.17 // indirect 98 | k8s.io/client-go v0.24.17 // indirect 99 | k8s.io/component-helpers v0.24.17 // indirect 100 | k8s.io/controller-manager v0.24.17 // indirect 101 | k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect 102 | k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect 103 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect 104 | sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect 105 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 106 | sigs.k8s.io/yaml v1.3.0 // indirect 107 | ) 108 | 109 | replace ( 110 | golang.org/x/sync => golang.org/x/sync v0.0.0-20181108010431-42b317875d0f 111 | golang.org/x/sys => golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba 112 | golang.org/x/tools => golang.org/x/tools v0.0.0-20190313210603-aa82965741a9 113 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.24.17 114 | ) 115 | 116 | replace k8s.io/kubectl => k8s.io/kubectl v0.24.17 117 | 118 | replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.24.17 119 | 120 | replace k8s.io/sample-controller => k8s.io/sample-controller v0.24.17 121 | 122 | replace k8s.io/component-helpers => k8s.io/component-helpers v0.24.17 123 | 124 | replace k8s.io/controller-manager => k8s.io/controller-manager v0.24.17 125 | 126 | replace k8s.io/mount-utils => k8s.io/mount-utils v0.24.17 127 | -------------------------------------------------------------------------------- /protocol_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | "testing" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | ) 28 | 29 | func TestLoadBalancerProtocol_CSProtocol(t *testing.T) { 30 | tests := []struct { 31 | name string 32 | protocol LoadBalancerProtocol 33 | want string 34 | }{ 35 | { 36 | name: "TCP protocol", 37 | protocol: LoadBalancerProtocolTCP, 38 | want: "tcp", 39 | }, 40 | { 41 | name: "UDP protocol", 42 | protocol: LoadBalancerProtocolUDP, 43 | want: "udp", 44 | }, 45 | { 46 | name: "TCP Proxy protocol", 47 | protocol: LoadBalancerProtocolTCPProxy, 48 | want: "tcp-proxy", 49 | }, 50 | { 51 | name: "Invalid protocol", 52 | protocol: LoadBalancerProtocolInvalid, 53 | want: "", 54 | }, 55 | { 56 | name: "Unknown protocol value", 57 | protocol: LoadBalancerProtocol(999), 58 | want: "", 59 | }, 60 | } 61 | 62 | for _, tt := range tests { 63 | t.Run(tt.name, func(t *testing.T) { 64 | if got := tt.protocol.CSProtocol(); got != tt.want { 65 | t.Errorf("CSProtocol() = %v, want %v", got, tt.want) 66 | } 67 | }) 68 | } 69 | } 70 | 71 | func TestLoadBalancerProtocol_IPProtocol(t *testing.T) { 72 | tests := []struct { 73 | name string 74 | protocol LoadBalancerProtocol 75 | want string 76 | }{ 77 | { 78 | name: "TCP protocol maps to tcp", 79 | protocol: LoadBalancerProtocolTCP, 80 | want: "tcp", 81 | }, 82 | { 83 | name: "TCP Proxy protocol also maps to tcp", 84 | protocol: LoadBalancerProtocolTCPProxy, 85 | want: "tcp", 86 | }, 87 | { 88 | name: "UDP protocol maps to udp", 89 | protocol: LoadBalancerProtocolUDP, 90 | want: "udp", 91 | }, 92 | { 93 | name: "Invalid protocol returns empty", 94 | protocol: LoadBalancerProtocolInvalid, 95 | want: "", 96 | }, 97 | { 98 | name: "Unknown protocol value returns empty", 99 | protocol: LoadBalancerProtocol(999), 100 | want: "", 101 | }, 102 | } 103 | 104 | for _, tt := range tests { 105 | t.Run(tt.name, func(t *testing.T) { 106 | if got := tt.protocol.IPProtocol(); got != tt.want { 107 | t.Errorf("IPProtocol() = %v, want %v", got, tt.want) 108 | } 109 | }) 110 | } 111 | } 112 | 113 | func TestLoadBalancerProtocol_String(t *testing.T) { 114 | // String() should return the same as CSProtocol() 115 | protocols := []LoadBalancerProtocol{ 116 | LoadBalancerProtocolTCP, 117 | LoadBalancerProtocolUDP, 118 | LoadBalancerProtocolTCPProxy, 119 | LoadBalancerProtocolInvalid, 120 | } 121 | 122 | for _, p := range protocols { 123 | if got, want := p.String(), p.CSProtocol(); got != want { 124 | t.Errorf("String() = %v, want %v (same as CSProtocol)", got, want) 125 | } 126 | } 127 | } 128 | 129 | func TestProtocolFromLoadBalancer(t *testing.T) { 130 | tests := []struct { 131 | name string 132 | protocol string 133 | want LoadBalancerProtocol 134 | }{ 135 | { 136 | name: "tcp string", 137 | protocol: "tcp", 138 | want: LoadBalancerProtocolTCP, 139 | }, 140 | { 141 | name: "udp string", 142 | protocol: "udp", 143 | want: LoadBalancerProtocolUDP, 144 | }, 145 | { 146 | name: "tcp-proxy string", 147 | protocol: "tcp-proxy", 148 | want: LoadBalancerProtocolTCPProxy, 149 | }, 150 | { 151 | name: "empty string returns invalid", 152 | protocol: "", 153 | want: LoadBalancerProtocolInvalid, 154 | }, 155 | { 156 | name: "unknown protocol returns invalid", 157 | protocol: "icmp", 158 | want: LoadBalancerProtocolInvalid, 159 | }, 160 | { 161 | name: "uppercase TCP returns invalid", 162 | protocol: "TCP", 163 | want: LoadBalancerProtocolInvalid, 164 | }, 165 | } 166 | 167 | for _, tt := range tests { 168 | t.Run(tt.name, func(t *testing.T) { 169 | if got := ProtocolFromLoadBalancer(tt.protocol); got != tt.want { 170 | t.Errorf("ProtocolFromLoadBalancer(%q) = %v, want %v", tt.protocol, got, tt.want) 171 | } 172 | }) 173 | } 174 | } 175 | 176 | func TestProtocolFromServicePort(t *testing.T) { 177 | tests := []struct { 178 | name string 179 | port corev1.ServicePort 180 | annotations map[string]string 181 | want LoadBalancerProtocol 182 | }{ 183 | { 184 | name: "TCP port without proxy annotation", 185 | port: corev1.ServicePort{ 186 | Protocol: corev1.ProtocolTCP, 187 | Port: 80, 188 | }, 189 | annotations: nil, 190 | want: LoadBalancerProtocolTCP, 191 | }, 192 | { 193 | name: "TCP port with proxy annotation true", 194 | port: corev1.ServicePort{ 195 | Protocol: corev1.ProtocolTCP, 196 | Port: 80, 197 | }, 198 | annotations: map[string]string{ 199 | ServiceAnnotationLoadBalancerProxyProtocol: "true", 200 | }, 201 | want: LoadBalancerProtocolTCPProxy, 202 | }, 203 | { 204 | name: "TCP port with proxy annotation false", 205 | port: corev1.ServicePort{ 206 | Protocol: corev1.ProtocolTCP, 207 | Port: 80, 208 | }, 209 | annotations: map[string]string{ 210 | ServiceAnnotationLoadBalancerProxyProtocol: "false", 211 | }, 212 | want: LoadBalancerProtocolTCP, 213 | }, 214 | { 215 | name: "UDP port", 216 | port: corev1.ServicePort{ 217 | Protocol: corev1.ProtocolUDP, 218 | Port: 53, 219 | }, 220 | annotations: nil, 221 | want: LoadBalancerProtocolUDP, 222 | }, 223 | { 224 | name: "UDP port ignores proxy annotation", 225 | port: corev1.ServicePort{ 226 | Protocol: corev1.ProtocolUDP, 227 | Port: 53, 228 | }, 229 | annotations: map[string]string{ 230 | ServiceAnnotationLoadBalancerProxyProtocol: "true", 231 | }, 232 | want: LoadBalancerProtocolUDP, 233 | }, 234 | { 235 | name: "SCTP port returns invalid", 236 | port: corev1.ServicePort{ 237 | Protocol: corev1.ProtocolSCTP, 238 | Port: 80, 239 | }, 240 | annotations: nil, 241 | want: LoadBalancerProtocolInvalid, 242 | }, 243 | } 244 | 245 | for _, tt := range tests { 246 | t.Run(tt.name, func(t *testing.T) { 247 | service := &corev1.Service{ 248 | ObjectMeta: metav1.ObjectMeta{ 249 | Name: "test-service", 250 | Namespace: "default", 251 | Annotations: tt.annotations, 252 | }, 253 | } 254 | if got := ProtocolFromServicePort(tt.port, service); got != tt.want { 255 | t.Errorf("ProtocolFromServicePort() = %v, want %v", got, tt.want) 256 | } 257 | }) 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /cloudstack_instances_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | "strings" 24 | "testing" 25 | 26 | "github.com/apache/cloudstack-go/v2/cloudstack" 27 | corev1 "k8s.io/api/core/v1" 28 | ) 29 | 30 | func TestNodeAddresses(t *testing.T) { 31 | cs := &CSCloud{} 32 | 33 | tests := []struct { 34 | name string 35 | instance *cloudstack.VirtualMachine 36 | wantAddrs []corev1.NodeAddress 37 | wantErr bool 38 | errContains string 39 | }{ 40 | { 41 | name: "instance with internal IP only", 42 | instance: &cloudstack.VirtualMachine{ 43 | Id: "vm-1", 44 | Name: "test-vm", 45 | Nic: []cloudstack.Nic{ 46 | {Ipaddress: "10.0.0.1"}, 47 | }, 48 | }, 49 | wantAddrs: []corev1.NodeAddress{ 50 | {Type: corev1.NodeInternalIP, Address: "10.0.0.1"}, 51 | }, 52 | wantErr: false, 53 | }, 54 | { 55 | name: "instance with internal IP and hostname", 56 | instance: &cloudstack.VirtualMachine{ 57 | Id: "vm-1", 58 | Name: "test-vm", 59 | Hostname: "test-hostname", 60 | Nic: []cloudstack.Nic{ 61 | {Ipaddress: "10.0.0.1"}, 62 | }, 63 | }, 64 | wantAddrs: []corev1.NodeAddress{ 65 | {Type: corev1.NodeInternalIP, Address: "10.0.0.1"}, 66 | {Type: corev1.NodeHostName, Address: "test-hostname"}, 67 | }, 68 | wantErr: false, 69 | }, 70 | { 71 | name: "instance with internal IP and public IP", 72 | instance: &cloudstack.VirtualMachine{ 73 | Id: "vm-1", 74 | Name: "test-vm", 75 | Publicip: "203.0.113.1", 76 | Nic: []cloudstack.Nic{ 77 | {Ipaddress: "10.0.0.1"}, 78 | }, 79 | }, 80 | wantAddrs: []corev1.NodeAddress{ 81 | {Type: corev1.NodeInternalIP, Address: "10.0.0.1"}, 82 | {Type: corev1.NodeExternalIP, Address: "203.0.113.1"}, 83 | }, 84 | wantErr: false, 85 | }, 86 | { 87 | name: "instance with all address types", 88 | instance: &cloudstack.VirtualMachine{ 89 | Id: "vm-1", 90 | Name: "test-vm", 91 | Hostname: "test-hostname", 92 | Publicip: "203.0.113.1", 93 | Nic: []cloudstack.Nic{ 94 | {Ipaddress: "10.0.0.1"}, 95 | }, 96 | }, 97 | wantAddrs: []corev1.NodeAddress{ 98 | {Type: corev1.NodeInternalIP, Address: "10.0.0.1"}, 99 | {Type: corev1.NodeHostName, Address: "test-hostname"}, 100 | {Type: corev1.NodeExternalIP, Address: "203.0.113.1"}, 101 | }, 102 | wantErr: false, 103 | }, 104 | { 105 | name: "instance with no NICs returns error", 106 | instance: &cloudstack.VirtualMachine{ 107 | Id: "vm-1", 108 | Name: "test-vm", 109 | Nic: []cloudstack.Nic{}, 110 | }, 111 | wantAddrs: nil, 112 | wantErr: true, 113 | errContains: "does not have an internal IP", 114 | }, 115 | { 116 | name: "instance with nil NICs returns error", 117 | instance: &cloudstack.VirtualMachine{ 118 | Id: "vm-1", 119 | Name: "test-vm", 120 | Nic: nil, 121 | }, 122 | wantAddrs: nil, 123 | wantErr: true, 124 | errContains: "does not have an internal IP", 125 | }, 126 | { 127 | name: "instance with multiple NICs uses first", 128 | instance: &cloudstack.VirtualMachine{ 129 | Id: "vm-1", 130 | Name: "test-vm", 131 | Nic: []cloudstack.Nic{ 132 | {Ipaddress: "10.0.0.1"}, 133 | {Ipaddress: "10.0.0.2"}, 134 | }, 135 | }, 136 | wantAddrs: []corev1.NodeAddress{ 137 | {Type: corev1.NodeInternalIP, Address: "10.0.0.1"}, 138 | }, 139 | wantErr: false, 140 | }, 141 | } 142 | 143 | for _, tt := range tests { 144 | t.Run(tt.name, func(t *testing.T) { 145 | gotAddrs, err := cs.nodeAddresses(tt.instance) 146 | 147 | if tt.wantErr { 148 | if err == nil { 149 | t.Errorf("nodeAddresses() expected error, got nil") 150 | return 151 | } 152 | if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) { 153 | t.Errorf("nodeAddresses() error = %v, want error containing %q", err, tt.errContains) 154 | } 155 | return 156 | } 157 | 158 | if err != nil { 159 | t.Errorf("nodeAddresses() unexpected error: %v", err) 160 | return 161 | } 162 | 163 | if len(gotAddrs) != len(tt.wantAddrs) { 164 | t.Errorf("nodeAddresses() returned %d addresses, want %d", len(gotAddrs), len(tt.wantAddrs)) 165 | return 166 | } 167 | 168 | for i, want := range tt.wantAddrs { 169 | if gotAddrs[i].Type != want.Type || gotAddrs[i].Address != want.Address { 170 | t.Errorf("nodeAddresses()[%d] = {%v, %v}, want {%v, %v}", 171 | i, gotAddrs[i].Type, gotAddrs[i].Address, want.Type, want.Address) 172 | } 173 | } 174 | }) 175 | } 176 | } 177 | 178 | func TestGetProviderIDFromInstanceID(t *testing.T) { 179 | cs := &CSCloud{} 180 | 181 | tests := []struct { 182 | name string 183 | instanceID string 184 | want string 185 | }{ 186 | { 187 | name: "valid instance ID", 188 | instanceID: "vm-123", 189 | want: "external-cloudstack://vm-123", 190 | }, 191 | { 192 | name: "empty instance ID", 193 | instanceID: "", 194 | want: "external-cloudstack://", 195 | }, 196 | } 197 | 198 | for _, tt := range tests { 199 | t.Run(tt.name, func(t *testing.T) { 200 | got := cs.getProviderIDFromInstanceID(tt.instanceID) 201 | if got != tt.want { 202 | t.Errorf("getProviderIDFromInstanceID(%q) = %q, want %q", tt.instanceID, got, tt.want) 203 | } 204 | }) 205 | } 206 | } 207 | 208 | func TestGetInstanceIDFromProviderID(t *testing.T) { 209 | cs := &CSCloud{} 210 | 211 | tests := []struct { 212 | name string 213 | providerID string 214 | want string 215 | }{ 216 | { 217 | name: "full provider ID format", 218 | providerID: "external-cloudstack://vm-123", 219 | want: "vm-123", 220 | }, 221 | { 222 | name: "instance ID only - backward compatibility", 223 | providerID: "vm-123", 224 | want: "vm-123", 225 | }, 226 | { 227 | name: "empty string", 228 | providerID: "", 229 | want: "", 230 | }, 231 | { 232 | name: "invalid format - no separator", 233 | providerID: "external-cloudstack-vm-123", 234 | want: "external-cloudstack-vm-123", 235 | }, 236 | { 237 | name: "different provider prefix", 238 | providerID: "aws://i-1234567890abcdef0", 239 | want: "i-1234567890abcdef0", 240 | }, 241 | } 242 | 243 | for _, tt := range tests { 244 | t.Run(tt.name, func(t *testing.T) { 245 | got := cs.getInstanceIDFromProviderID(tt.providerID) 246 | if got != tt.want { 247 | t.Errorf("getInstanceIDFromProviderID(%q) = %q, want %q", tt.providerID, got, tt.want) 248 | } 249 | }) 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /cloudstack_instances.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | "context" 24 | "errors" 25 | "fmt" 26 | "regexp" 27 | "strings" 28 | 29 | "github.com/apache/cloudstack-go/v2/cloudstack" 30 | corev1 "k8s.io/api/core/v1" 31 | "k8s.io/apimachinery/pkg/types" 32 | cloudprovider "k8s.io/cloud-provider" 33 | "k8s.io/klog/v2" 34 | ) 35 | 36 | var labelInvalidCharsRegex *regexp.Regexp = regexp.MustCompile(`([^A-Za-z0-9][^-A-Za-z0-9_.]*)?[^A-Za-z0-9]`) 37 | 38 | // NodeAddresses returns the addresses of the specified instance. 39 | func (cs *CSCloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]corev1.NodeAddress, error) { 40 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName( 41 | string(name), 42 | cloudstack.WithProject(cs.projectID), 43 | ) 44 | if err != nil { 45 | if count == 0 { 46 | return nil, cloudprovider.InstanceNotFound 47 | } 48 | return nil, fmt.Errorf("error retrieving node addresses: %v", err) 49 | } 50 | 51 | return cs.nodeAddresses(instance) 52 | } 53 | 54 | // NodeAddressesByProviderID returns the addresses of the specified instance. 55 | func (cs *CSCloud) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]corev1.NodeAddress, error) { 56 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID( 57 | cs.getInstanceIDFromProviderID(providerID), 58 | cloudstack.WithProject(cs.projectID), 59 | ) 60 | if err != nil { 61 | if count == 0 { 62 | return nil, cloudprovider.InstanceNotFound 63 | } 64 | return nil, fmt.Errorf("error retrieving node addresses: %v", err) 65 | } 66 | 67 | return cs.nodeAddresses(instance) 68 | } 69 | 70 | func (cs *CSCloud) nodeAddresses(instance *cloudstack.VirtualMachine) ([]corev1.NodeAddress, error) { 71 | if len(instance.Nic) == 0 { 72 | return nil, errors.New("instance does not have an internal IP") 73 | } 74 | 75 | addresses := []corev1.NodeAddress{ 76 | {Type: corev1.NodeInternalIP, Address: instance.Nic[0].Ipaddress}, 77 | } 78 | 79 | if instance.Hostname != "" { 80 | addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: instance.Hostname}) 81 | } 82 | 83 | if instance.Publicip != "" { 84 | addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: instance.Publicip}) 85 | } else { 86 | // Since there is no sane way to determine the external IP if the host isn't 87 | // using static NAT, we will just fire a log message and omit the external IP. 88 | klog.V(4).Infof("Could not determine the public IP of host %v (%v)", instance.Name, instance.Id) 89 | } 90 | 91 | return addresses, nil 92 | } 93 | 94 | // InstanceID returns the cloud provider ID of the specified instance. 95 | func (cs *CSCloud) InstanceID(ctx context.Context, name types.NodeName) (string, error) { 96 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName( 97 | string(name), 98 | cloudstack.WithProject(cs.projectID), 99 | ) 100 | if err != nil { 101 | if count == 0 { 102 | return "", cloudprovider.InstanceNotFound 103 | } 104 | return "", fmt.Errorf("error retrieving instance ID: %v", err) 105 | } 106 | 107 | return instance.Id, nil 108 | } 109 | 110 | // InstanceType returns the type of the specified instance. 111 | func (cs *CSCloud) InstanceType(ctx context.Context, name types.NodeName) (string, error) { 112 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName( 113 | string(name), 114 | cloudstack.WithProject(cs.projectID), 115 | ) 116 | if err != nil { 117 | if count == 0 { 118 | return "", cloudprovider.InstanceNotFound 119 | } 120 | return "", fmt.Errorf("error retrieving instance type: %v", err) 121 | } 122 | 123 | return labelInvalidCharsRegex.ReplaceAllString(instance.Serviceofferingname, ``), nil 124 | } 125 | 126 | // InstanceTypeByProviderID returns the type of the specified instance. 127 | func (cs *CSCloud) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) { 128 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID( 129 | cs.getInstanceIDFromProviderID(providerID), 130 | cloudstack.WithProject(cs.projectID), 131 | ) 132 | if err != nil { 133 | if count == 0 { 134 | return "", cloudprovider.InstanceNotFound 135 | } 136 | return "", fmt.Errorf("error retrieving instance type: %v", err) 137 | } 138 | 139 | return labelInvalidCharsRegex.ReplaceAllString(instance.Serviceofferingname, ``), nil 140 | } 141 | 142 | // AddSSHKeyToAllInstances is currently not implemented. 143 | func (cs *CSCloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error { 144 | return cloudprovider.NotImplemented 145 | } 146 | 147 | // CurrentNodeName returns the name of the node we are currently running on. 148 | func (cs *CSCloud) CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error) { 149 | return types.NodeName(hostname), nil 150 | } 151 | 152 | // InstanceExistsByProviderID returns if the instance still exists. 153 | func (cs *CSCloud) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) { 154 | _, count, err := cs.client.VirtualMachine.GetVirtualMachineByID( 155 | cs.getInstanceIDFromProviderID(providerID), 156 | cloudstack.WithProject(cs.projectID), 157 | ) 158 | if err != nil { 159 | if count == 0 { 160 | return false, nil 161 | } 162 | return false, fmt.Errorf("error retrieving instance: %v", err) 163 | } 164 | 165 | return true, nil 166 | } 167 | 168 | // InstanceShutdownByProviderID returns true if the instance is in safe state to detach volumes 169 | func (cs *CSCloud) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) { 170 | return false, cloudprovider.NotImplemented 171 | } 172 | 173 | func (cs *CSCloud) InstanceExists(ctx context.Context, node *corev1.Node) (bool, error) { 174 | nodeName := types.NodeName(node.Name) 175 | providerID, err := cs.InstanceID(ctx, nodeName) 176 | if err != nil { 177 | return false, err 178 | } 179 | 180 | return cs.InstanceExistsByProviderID(ctx, providerID) 181 | } 182 | 183 | func (cs *CSCloud) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) { 184 | return false, cloudprovider.NotImplemented 185 | } 186 | 187 | func (cs *CSCloud) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) { 188 | 189 | instanceID, err := cs.InstanceID(ctx, types.NodeName(node.Name)) 190 | if err != nil { 191 | return nil, err 192 | } 193 | 194 | instanceType, err := cs.InstanceType(ctx, types.NodeName(node.Name)) 195 | if err != nil { 196 | return nil, err 197 | } 198 | 199 | addresses, err := cs.NodeAddresses(ctx, types.NodeName(node.Name)) 200 | if err != nil { 201 | return nil, err 202 | } 203 | 204 | zone, err := cs.GetZoneByNodeName(ctx, types.NodeName(node.Name)) 205 | if err != nil { 206 | return nil, err 207 | } 208 | 209 | return &cloudprovider.InstanceMetadata{ 210 | ProviderID: cs.getProviderIDFromInstanceID(instanceID), 211 | InstanceType: instanceType, 212 | NodeAddresses: addresses, 213 | Zone: zone.FailureDomain, 214 | Region: zone.Region, 215 | }, nil 216 | } 217 | 218 | func (cs *CSCloud) getProviderIDFromInstanceID(instanceID string) string { 219 | return fmt.Sprintf("%s://%s", cs.ProviderName(), instanceID) 220 | } 221 | 222 | func (cs *CSCloud) getInstanceIDFromProviderID(providerID string) string { 223 | parts := strings.Split(providerID, "://") 224 | if len(parts) == 1 { 225 | return providerID 226 | } 227 | return parts[1] 228 | } 229 | -------------------------------------------------------------------------------- /cloudstack_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | "context" 24 | "errors" 25 | "os" 26 | "strconv" 27 | "strings" 28 | "testing" 29 | 30 | "github.com/apache/cloudstack-go/v2/cloudstack" 31 | "github.com/blang/semver/v4" 32 | "go.uber.org/mock/gomock" 33 | corev1 "k8s.io/api/core/v1" 34 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 | ) 36 | 37 | const testClusterName = "testCluster" 38 | 39 | func TestReadConfig(t *testing.T) { 40 | _, err := readConfig(nil) 41 | if err != nil { 42 | t.Fatalf("Should not return an error when no config is provided: %v", err) 43 | } 44 | 45 | cfg, err := readConfig(strings.NewReader(` 46 | [Global] 47 | api-url = https://cloudstack.url 48 | api-key = a-valid-api-key 49 | secret-key = a-valid-secret-key 50 | ssl-no-verify = true 51 | project-id = a-valid-project-id 52 | `)) 53 | if err != nil { 54 | t.Fatalf("Should succeed when a valid config is provided: %v", err) 55 | } 56 | 57 | if cfg.Global.APIURL != "https://cloudstack.url" { 58 | t.Errorf("incorrect api-url: %s", cfg.Global.APIURL) 59 | } 60 | if cfg.Global.APIKey != "a-valid-api-key" { 61 | t.Errorf("incorrect api-key: %s", cfg.Global.APIKey) 62 | } 63 | if cfg.Global.SecretKey != "a-valid-secret-key" { 64 | t.Errorf("incorrect secret-key: %s", cfg.Global.SecretKey) 65 | } 66 | if !cfg.Global.SSLNoVerify { 67 | t.Errorf("incorrect ssl-no-verify: %t", cfg.Global.SSLNoVerify) 68 | } 69 | } 70 | 71 | // This allows acceptance testing against an existing CloudStack environment. 72 | func configFromEnv() (*CSConfig, bool) { 73 | cfg := &CSConfig{} 74 | 75 | cfg.Global.APIURL = os.Getenv("CS_API_URL") 76 | cfg.Global.APIKey = os.Getenv("CS_API_KEY") 77 | cfg.Global.SecretKey = os.Getenv("CS_SECRET_KEY") 78 | cfg.Global.ProjectID = os.Getenv("CS_PROJECT_ID") 79 | 80 | // It is save to ignore the error here. If the input cannot be parsed SSLNoVerify 81 | // will still be a bool with its zero value (false) which is the expected default. 82 | cfg.Global.SSLNoVerify, _ = strconv.ParseBool(os.Getenv("CS_SSL_NO_VERIFY")) 83 | 84 | // Check if we have the minimum required info to be able to connect to CloudStack. 85 | ok := cfg.Global.APIURL != "" && cfg.Global.APIKey != "" && cfg.Global.SecretKey != "" 86 | 87 | return cfg, ok 88 | } 89 | 90 | func TestNewCSCloud(t *testing.T) { 91 | cfg, ok := configFromEnv() 92 | if !ok { 93 | t.Skipf("No config found in environment") 94 | } 95 | 96 | _, err := newCSCloud(cfg) 97 | if err != nil { 98 | t.Fatalf("Failed to construct/authenticate CloudStack: %v", err) 99 | } 100 | } 101 | 102 | func TestLoadBalancer(t *testing.T) { 103 | cfg, ok := configFromEnv() 104 | if !ok { 105 | t.Skipf("No config found in environment") 106 | } 107 | 108 | cs, err := newCSCloud(cfg) 109 | if err != nil { 110 | t.Fatalf("Failed to construct/authenticate CloudStack: %v", err) 111 | } 112 | 113 | lb, ok := cs.LoadBalancer() 114 | if !ok { 115 | t.Fatalf("LoadBalancer() returned false") 116 | } 117 | 118 | _, exists, err := lb.GetLoadBalancer(context.TODO(), testClusterName, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}}) 119 | if err != nil { 120 | t.Fatalf("GetLoadBalancer(\"noexist\") returned error: %s", err) 121 | } 122 | if exists { 123 | t.Fatalf("GetLoadBalancer(\"noexist\") returned exists") 124 | } 125 | } 126 | 127 | func TestGetManagementServerVersion(t *testing.T) { 128 | t.Run("returns parsed version", func(t *testing.T) { 129 | ctrl := gomock.NewController(t) 130 | t.Cleanup(ctrl.Finish) 131 | 132 | mockMgmt := cloudstack.NewMockManagementServiceIface(ctrl) 133 | params := &cloudstack.ListManagementServersMetricsParams{} 134 | resp := &cloudstack.ListManagementServersMetricsResponse{ 135 | Count: 1, 136 | ManagementServersMetrics: []*cloudstack.ManagementServersMetric{ 137 | {Version: "4.17.1.0"}, 138 | }, 139 | } 140 | 141 | gomock.InOrder( 142 | mockMgmt.EXPECT().NewListManagementServersMetricsParams().Return(params), 143 | mockMgmt.EXPECT().ListManagementServersMetrics(params).Return(resp, nil), 144 | ) 145 | 146 | cs := &CSCloud{ 147 | client: &cloudstack.CloudStackClient{ 148 | Management: mockMgmt, 149 | }, 150 | } 151 | 152 | version, err := cs.getManagementServerVersion() 153 | if err != nil { 154 | t.Fatalf("unexpected error: %v", err) 155 | } 156 | 157 | expected := semver.MustParse("4.17.1") 158 | if !version.Equals(expected) { 159 | t.Fatalf("version = %v, want %v", version, expected) 160 | } 161 | }) 162 | 163 | t.Run("returns correct parsed version with development server", func(t *testing.T) { 164 | ctrl := gomock.NewController(t) 165 | t.Cleanup(ctrl.Finish) 166 | 167 | mockMgmt := cloudstack.NewMockManagementServiceIface(ctrl) 168 | params := &cloudstack.ListManagementServersMetricsParams{} 169 | resp := &cloudstack.ListManagementServersMetricsResponse{ 170 | Count: 1, 171 | ManagementServersMetrics: []*cloudstack.ManagementServersMetric{ 172 | {Version: "4.17.1.0-SNAPSHOT"}, 173 | }, 174 | } 175 | 176 | gomock.InOrder( 177 | mockMgmt.EXPECT().NewListManagementServersMetricsParams().Return(params), 178 | mockMgmt.EXPECT().ListManagementServersMetrics(params).Return(resp, nil), 179 | ) 180 | 181 | cs := &CSCloud{ 182 | client: &cloudstack.CloudStackClient{ 183 | Management: mockMgmt, 184 | }, 185 | } 186 | 187 | version, err := cs.getManagementServerVersion() 188 | if err != nil { 189 | t.Fatalf("unexpected error: %v", err) 190 | } 191 | 192 | expected := semver.MustParse("4.17.1") 193 | if !version.Equals(expected) { 194 | t.Fatalf("version = %v, want %v", version, expected) 195 | } 196 | }) 197 | 198 | t.Run("returns error when api call fails", func(t *testing.T) { 199 | ctrl := gomock.NewController(t) 200 | t.Cleanup(ctrl.Finish) 201 | 202 | mockMgmt := cloudstack.NewMockManagementServiceIface(ctrl) 203 | params := &cloudstack.ListManagementServersMetricsParams{} 204 | apiErr := errors.New("api failure") 205 | 206 | gomock.InOrder( 207 | mockMgmt.EXPECT().NewListManagementServersMetricsParams().Return(params), 208 | mockMgmt.EXPECT().ListManagementServersMetrics(params).Return(nil, apiErr), 209 | ) 210 | 211 | cs := &CSCloud{ 212 | client: &cloudstack.CloudStackClient{ 213 | Management: mockMgmt, 214 | }, 215 | } 216 | 217 | if _, err := cs.getManagementServerVersion(); err == nil { 218 | t.Fatalf("expected error, got nil") 219 | } 220 | }) 221 | 222 | t.Run("returns error when no servers found", func(t *testing.T) { 223 | ctrl := gomock.NewController(t) 224 | t.Cleanup(ctrl.Finish) 225 | 226 | mockMgmt := cloudstack.NewMockManagementServiceIface(ctrl) 227 | params := &cloudstack.ListManagementServersMetricsParams{} 228 | resp := &cloudstack.ListManagementServersMetricsResponse{ 229 | Count: 0, 230 | ManagementServersMetrics: []*cloudstack.ManagementServersMetric{}, 231 | } 232 | 233 | gomock.InOrder( 234 | mockMgmt.EXPECT().NewListManagementServersMetricsParams().Return(params), 235 | mockMgmt.EXPECT().ListManagementServersMetrics(params).Return(resp, nil), 236 | ) 237 | 238 | cs := &CSCloud{ 239 | client: &cloudstack.CloudStackClient{ 240 | Management: mockMgmt, 241 | }, 242 | } 243 | 244 | if _, err := cs.getManagementServerVersion(); err == nil { 245 | t.Fatalf("expected error for zero management servers") 246 | } 247 | }) 248 | 249 | t.Run("returns error when version cannot be parsed", func(t *testing.T) { 250 | ctrl := gomock.NewController(t) 251 | t.Cleanup(ctrl.Finish) 252 | 253 | mockMgmt := cloudstack.NewMockManagementServiceIface(ctrl) 254 | params := &cloudstack.ListManagementServersMetricsParams{} 255 | resp := &cloudstack.ListManagementServersMetricsResponse{ 256 | Count: 1, 257 | ManagementServersMetrics: []*cloudstack.ManagementServersMetric{ 258 | {Version: "invalid.version.string"}, 259 | }, 260 | } 261 | 262 | gomock.InOrder( 263 | mockMgmt.EXPECT().NewListManagementServersMetricsParams().Return(params), 264 | mockMgmt.EXPECT().ListManagementServersMetrics(params).Return(resp, nil), 265 | ) 266 | 267 | cs := &CSCloud{ 268 | client: &cloudstack.CloudStackClient{ 269 | Management: mockMgmt, 270 | }, 271 | } 272 | 273 | if _, err := cs.getManagementServerVersion(); err == nil { 274 | t.Fatalf("expected parse error") 275 | } 276 | }) 277 | } 278 | 279 | func TestGetRegionFromZone(t *testing.T) { 280 | tests := []struct { 281 | name string 282 | region string 283 | zone string 284 | want string 285 | }{ 286 | { 287 | name: "region configured in cloud config", 288 | region: "us-east-1", 289 | zone: "zone-1", 290 | want: "us-east-1", 291 | }, 292 | { 293 | name: "region not configured, returns zone", 294 | region: "", 295 | zone: "zone-1", 296 | want: "zone-1", 297 | }, 298 | { 299 | name: "region configured with empty zone", 300 | region: "eu-central-1", 301 | zone: "", 302 | want: "eu-central-1", 303 | }, 304 | } 305 | 306 | for _, tt := range tests { 307 | t.Run(tt.name, func(t *testing.T) { 308 | cs := &CSCloud{ 309 | region: tt.region, 310 | } 311 | got := cs.getRegionFromZone(tt.zone) 312 | if got != tt.want { 313 | t.Errorf("getRegionFromZone(%q) with region=%q = %q, want %q", tt.zone, tt.region, got, tt.want) 314 | } 315 | }) 316 | } 317 | } 318 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 The Apache Software Foundation 2 | 3 | 4 | Apache License 5 | Version 2.0, January 2004 6 | http://www.apache.org/licenses/ 7 | 8 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 9 | 10 | 1. Definitions. 11 | 12 | "License" shall mean the terms and conditions for use, reproduction, 13 | and distribution as defined by Sections 1 through 9 of this document. 14 | 15 | "Licensor" shall mean the copyright owner or entity authorized by 16 | the copyright owner that is granting the License. 17 | 18 | "Legal Entity" shall mean the union of the acting entity and all 19 | other entities that control, are controlled by, or are under common 20 | control with that entity. For the purposes of this definition, 21 | "control" means (i) the power, direct or indirect, to cause the 22 | direction or management of such entity, whether by contract or 23 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 24 | outstanding shares, or (iii) beneficial ownership of such entity. 25 | 26 | "You" (or "Your") shall mean an individual or Legal Entity 27 | exercising permissions granted by this License. 28 | 29 | "Source" form shall mean the preferred form for making modifications, 30 | including but not limited to software source code, documentation 31 | source, and configuration files. 32 | 33 | "Object" form shall mean any form resulting from mechanical 34 | transformation or translation of a Source form, including but 35 | not limited to compiled object code, generated documentation, 36 | and conversions to other media types. 37 | 38 | "Work" shall mean the work of authorship, whether in Source or 39 | Object form, made available under the License, as indicated by a 40 | copyright notice that is included in or attached to the work 41 | (an example is provided in the Appendix below). 42 | 43 | "Derivative Works" shall mean any work, whether in Source or Object 44 | form, that is based on (or derived from) the Work and for which the 45 | editorial revisions, annotations, elaborations, or other modifications 46 | represent, as a whole, an original work of authorship. For the purposes 47 | of this License, Derivative Works shall not include works that remain 48 | separable from, or merely link (or bind by name) to the interfaces of, 49 | the Work and Derivative Works thereof. 50 | 51 | "Contribution" shall mean any work of authorship, including 52 | the original version of the Work and any modifications or additions 53 | to that Work or Derivative Works thereof, that is intentionally 54 | submitted to Licensor for inclusion in the Work by the copyright owner 55 | or by an individual or Legal Entity authorized to submit on behalf of 56 | the copyright owner. For the purposes of this definition, "submitted" 57 | means any form of electronic, verbal, or written communication sent 58 | to the Licensor or its representatives, including but not limited to 59 | communication on electronic mailing lists, source code control systems, 60 | and issue tracking systems that are managed by, or on behalf of, the 61 | Licensor for the purpose of discussing and improving the Work, but 62 | excluding communication that is conspicuously marked or otherwise 63 | designated in writing by the copyright owner as "Not a Contribution." 64 | 65 | "Contributor" shall mean Licensor and any individual or Legal Entity 66 | on behalf of whom a Contribution has been received by Licensor and 67 | subsequently incorporated within the Work. 68 | 69 | 2. Grant of Copyright License. Subject to the terms and conditions of 70 | this License, each Contributor hereby grants to You a perpetual, 71 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 72 | copyright license to reproduce, prepare Derivative Works of, 73 | publicly display, publicly perform, sublicense, and distribute the 74 | Work and such Derivative Works in Source or Object form. 75 | 76 | 3. Grant of Patent License. Subject to the terms and conditions of 77 | this License, each Contributor hereby grants to You a perpetual, 78 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 79 | (except as stated in this section) patent license to make, have made, 80 | use, offer to sell, sell, import, and otherwise transfer the Work, 81 | where such license applies only to those patent claims licensable 82 | by such Contributor that are necessarily infringed by their 83 | Contribution(s) alone or by combination of their Contribution(s) 84 | with the Work to which such Contribution(s) was submitted. If You 85 | institute patent litigation against any entity (including a 86 | cross-claim or counterclaim in a lawsuit) alleging that the Work 87 | or a Contribution incorporated within the Work constitutes direct 88 | or contributory patent infringement, then any patent licenses 89 | granted to You under this License for that Work shall terminate 90 | as of the date such litigation is filed. 91 | 92 | 4. Redistribution. You may reproduce and distribute copies of the 93 | Work or Derivative Works thereof in any medium, with or without 94 | modifications, and in Source or Object form, provided that You 95 | meet the following conditions: 96 | 97 | (a) You must give any other recipients of the Work or 98 | Derivative Works a copy of this License; and 99 | 100 | (b) You must cause any modified files to carry prominent notices 101 | stating that You changed the files; and 102 | 103 | (c) You must retain, in the Source form of any Derivative Works 104 | that You distribute, all copyright, patent, trademark, and 105 | attribution notices from the Source form of the Work, 106 | excluding those notices that do not pertain to any part of 107 | the Derivative Works; and 108 | 109 | (d) If the Work includes a "NOTICE" text file as part of its 110 | distribution, then any Derivative Works that You distribute must 111 | include a readable copy of the attribution notices contained 112 | within such NOTICE file, excluding those notices that do not 113 | pertain to any part of the Derivative Works, in at least one 114 | of the following places: within a NOTICE text file distributed 115 | as part of the Derivative Works; within the Source form or 116 | documentation, if provided along with the Derivative Works; or, 117 | within a display generated by the Derivative Works, if and 118 | wherever such third-party notices normally appear. The contents 119 | of the NOTICE file are for informational purposes only and 120 | do not modify the License. You may add Your own attribution 121 | notices within Derivative Works that You distribute, alongside 122 | or as an addendum to the NOTICE text from the Work, provided 123 | that such additional attribution notices cannot be construed 124 | as modifying the License. 125 | 126 | You may add Your own copyright statement to Your modifications and 127 | may provide additional or different license terms and conditions 128 | for use, reproduction, or distribution of Your modifications, or 129 | for any such Derivative Works as a whole, provided Your use, 130 | reproduction, and distribution of the Work otherwise complies with 131 | the conditions stated in this License. 132 | 133 | 5. Submission of Contributions. Unless You explicitly state otherwise, 134 | any Contribution intentionally submitted for inclusion in the Work 135 | by You to the Licensor shall be under the terms and conditions of 136 | this License, without any additional terms or conditions. 137 | Notwithstanding the above, nothing herein shall supersede or modify 138 | the terms of any separate license agreement you may have executed 139 | with Licensor regarding such Contributions. 140 | 141 | 6. Trademarks. This License does not grant permission to use the trade 142 | names, trademarks, service marks, or product names of the Licensor, 143 | except as required for reasonable and customary use in describing the 144 | origin of the Work and reproducing the content of the NOTICE file. 145 | 146 | 7. Disclaimer of Warranty. Unless required by applicable law or 147 | agreed to in writing, Licensor provides the Work (and each 148 | Contributor provides its Contributions) on an "AS IS" BASIS, 149 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 150 | implied, including, without limitation, any warranties or conditions 151 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 152 | PARTICULAR PURPOSE. You are solely responsible for determining the 153 | appropriateness of using or redistributing the Work and assume any 154 | risks associated with Your exercise of permissions under this License. 155 | 156 | 8. Limitation of Liability. In no event and under no legal theory, 157 | whether in tort (including negligence), contract, or otherwise, 158 | unless required by applicable law (such as deliberate and grossly 159 | negligent acts) or agreed to in writing, shall any Contributor be 160 | liable to You for damages, including any direct, indirect, special, 161 | incidental, or consequential damages of any character arising as a 162 | result of this License or out of the use or inability to use the 163 | Work (including but not limited to damages for loss of goodwill, 164 | work stoppage, computer failure or malfunction, or any and all 165 | other commercial damages or losses), even if such Contributor 166 | has been advised of the possibility of such damages. 167 | 168 | 9. Accepting Warranty or Additional Liability. While redistributing 169 | the Work or Derivative Works thereof, You may choose to offer, 170 | and charge a fee for, acceptance of support, warranty, indemnity, 171 | or other liability obligations and/or rights consistent with this 172 | License. However, in accepting such obligations, You may act only 173 | on Your own behalf and on Your sole responsibility, not on behalf 174 | of any other Contributor, and only if You agree to indemnify, 175 | defend, and hold each Contributor harmless for any liability 176 | incurred by, or claims asserted against, such Contributor by reason 177 | of your accepting any such warranty or additional liability. 178 | 179 | END OF TERMS AND CONDITIONS 180 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CloudStack Kubernetes Provider 2 | 3 | [![](https://img.shields.io/github/release/apache/cloudstack-kubernetes-provider.svg?logo=github&style=flat-square "Release")](https://github.com/apache/cloudstack-kubernetes-provider/releases) 4 | [![](https://img.shields.io/badge/license-Apache%202.0-blue.svg?color=%23282661&logo=apache&style=flat-square "Apache 2.0 license")](/LICENSE-2.0) 5 | [![](https://img.shields.io/badge/language-Go-%235adaff.svg?logo=go&style=flat-square "Go language")](https://golang.org) 6 | [![](https://img.shields.io/docker/v/apache/cloudstack-kubernetes-provider?label=docker%20hub&logo=docker&style=flat-square "Docker Hub Image Version")](https://hub.docker.com/r/apache/cloudstack-kubernetes-provider/) 7 | 8 | A Cloud Controller Manager to facilitate Kubernetes deployments on Cloudstack. 9 | 10 | Based on the old Cloudstack provider in Kubernetes was removed. 11 | 12 | Refer: 13 | * https://github.com/kubernetes/kubernetes/tree/release-1.15/pkg/cloudprovider/providers/cloudstack 14 | * https://github.com/kubernetes/enhancements/issues/672 15 | * https://github.com/kubernetes/enhancements/issues/88 16 | 17 | ## Deployment 18 | 19 | The CloudStack Kubernetes Provider is automatically deployed when a Kubernetes Cluster is created on CloudStack 4.16+ 20 | 21 | In order to communicate with CloudStack, a separate service user **kubeadmin** is created in the same account as the cluster owner. 22 | The provider uses this user's API keys to get the details of the cluster as well as update the networking rules. It is imperative that this user 23 | is not altered or have its keys regenerated. 24 | 25 | The provider can also be manually deployed as follows : 26 | 27 | ### Kubernetes 28 | 29 | Prebuilt containers are posted on [Docker Hub](https://hub.docker.com/r/apache/cloudstack-kubernetes-provider). 30 | 31 | To configure API access to your CloudStack management server, you need to create a secret containing a `cloud-config` 32 | that is suitable for your environment. 33 | 34 | `cloud-config` should look like this: 35 | ```ini 36 | [Global] 37 | api-url = 38 | api-key = 39 | secret-key = 40 | project-id = 41 | zone = 42 | ssl-no-verify = 43 | ``` 44 | 45 | The access token needs to be able to fetch VM information and deploy load balancers in the project or domain where the nodes reside. 46 | 47 | To create the secret, use the following command: 48 | ```bash 49 | kubectl -n kube-system create secret generic cloudstack-secret --from-file=cloud-config 50 | ``` 51 | 52 | You can then use the provided example [deployment.yaml](/deployment.yaml) to deploy the controller: 53 | ```bash 54 | kubectl apply -f deployment.yaml 55 | ``` 56 | 57 | ### Protocols 58 | 59 | This CCM supports TCP, UDP and [TCP-Proxy](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) LoadBalancer deployments. 60 | 61 | For UDP and Proxy Protocol support, CloudStack 4.6 or later is required. 62 | 63 | Since kube-proxy does not support the Proxy Protocol or UDP, you should connect this directly to pods, for example by deploying a DaemonSet and setting `hostPort: ` on the desired container port. 64 | Important: The service running in the pod must support the chosen protocol. Do not try to enable TCP-Proxy when the service only supports regular TCP. 65 | 66 | [traefik-ingress-controller.yml](/traefik-ingress-controller.yml) contains a basic deployment for the Træfik ingress controller that illustrates how to use it with the proxy protocol. 67 | 68 | For the nginx ingress controller, please refer to the official documentation at [kubernetes.github.io/ingress-nginx/deploy](https://kubernetes.github.io/ingress-nginx/deploy/). After applying the deployment, patch it for proxy protocol support with the provided fragment: 69 | 70 | ```bash 71 | kubectl apply -f nginx-ingress-controller-patch.yml 72 | ``` 73 | 74 | ### Service Annotations 75 | 76 | The CloudStack Kubernetes Provider supports several annotations on LoadBalancer services to customize load balancer behavior: 77 | 78 | #### `service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol` 79 | 80 | **Type:** Boolean (`"true"` or `"false"`) 81 | 82 | **Default:** `false` 83 | 84 | **Description:** Enables the [HAProxy Proxy Protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on a CloudStack load balancer. This annotation only applies to TCP service ports and requires CloudStack 4.6 or later. 85 | 86 | **Use Case:** Use this annotation when you need to preserve the original client IP address through the load balancer. This is commonly required for ingress controllers like Traefik or Nginx that need to know the client's real IP address. 87 | 88 | **Example:** 89 | ```yaml 90 | apiVersion: v1 91 | kind: Service 92 | metadata: 93 | name: my-service 94 | annotations: 95 | service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol: "true" 96 | spec: 97 | type: LoadBalancer 98 | ports: 99 | - port: 80 100 | protocol: TCP 101 | ``` 102 | 103 | #### `service.beta.kubernetes.io/cloudstack-load-balancer-hostname` 104 | 105 | **Type:** String 106 | 107 | **Default:** Not set (uses IP address) 108 | 109 | **Description:** Sets a hostname for the load balancer ingress instead of using the IP address. This is a workaround for [Kubernetes issue #66607](https://github.com/kubernetes/kubernetes/issues/66607). 110 | 111 | **Use Case:** Use this annotation when you need the LoadBalancer status to return a hostname instead of an IP address. This is useful for DNS-based routing or when you want to expose a specific hostname. 112 | 113 | **Example:** 114 | ```yaml 115 | apiVersion: v1 116 | kind: Service 117 | metadata: 118 | name: my-service 119 | annotations: 120 | service.beta.kubernetes.io/cloudstack-load-balancer-hostname: "lb.example.com" 121 | spec: 122 | type: LoadBalancer 123 | ``` 124 | 125 | 126 | #### `service.beta.kubernetes.io/cloudstack-load-balancer-source-cidrs` 127 | 128 | **Type:** String (comma-separated CIDR list) 129 | 130 | **Default:** `"0.0.0.0/0"` (allows all sources) 131 | 132 | **Description:** Specifies the source CIDR list for firewall rules on the CloudStack load balancer. This restricts which IP addresses can access the load balancer. 133 | 134 | **Use Case:** Use this annotation to restrict access to your load balancer to specific IP ranges for security purposes. This is particularly useful for internal services or when you want to limit access to specific networks. 135 | 136 | **Example:** 137 | ```yaml 138 | apiVersion: v1 139 | kind: Service 140 | metadata: 141 | name: my-service 142 | annotations: 143 | service.beta.kubernetes.io/cloudstack-load-balancer-source-cidrs: "10.0.0.0/8,192.168.1.0/24" 144 | spec: 145 | type: LoadBalancer 146 | ``` 147 | 148 | **Format:** Comma-separated list of CIDR ranges. Spaces around commas are automatically trimmed. 149 | 150 | **CloudStack Version:** Updating CIDR lists on existing load balancer rules requires CloudStack 4.22 or later. Creating new load balancer rules with CIDR lists works on earlier versions. 151 | 152 | **Note:** If the annotation is not set, the default behavior is to allow all sources (`0.0.0.0/0`). However, if you explicitly set the annotation to an empty value (`""`), this will result in an empty CIDR list, effectively blocking all traffic. 153 | 154 | ### Node Labels 155 | 156 | :warning: **The node name must match the host name, so the controller can fetch and assign metadata from CloudStack.** 157 | 158 | It is recommended to launch `kubelet` with the following parameter: 159 | 160 | ``` 161 | --register-with-taints=node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule 162 | ``` 163 | 164 | This will treat the node as 'uninitialized' and cause the CCM to apply metadata labels from CloudStack automatically. 165 | 166 | Supported labels for Kubernetes versions up to 1.16 are: 167 | * kubernetes.io/hostname (= the instance name) 168 | * beta.kubernetes.io/instance-type (= the compute offering) 169 | * failure-domain.beta.kubernetes.io/zone (= the zone) 170 | * failure-domain.beta.kubernetes.io/region (= region from config if defined, otherwise the zone) 171 | 172 | Supported labels for Kubernetes versions 1.17 and later are: 173 | * kubernetes.io/hostname (= the instance name) 174 | * node.kubernetes.io/instance-type (= the compute offering) 175 | * topology.kubernetes.io/zone (= the zone) 176 | * topology.kubernetes.io/region (= region from config if defined, otherwise the zone) 177 | 178 | It is also possible to trigger this process manually by issuing the following command: 179 | 180 | ``` 181 | kubectl taint nodes node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule 182 | ``` 183 | 184 | ## Migration Guide 185 | 186 | There are several notable differences to the old Kubernetes CloudStack cloud provider that need to be taken into 187 | account when migrating from the old cloud provider to the standalone controller. 188 | 189 | ### Load Balancer 190 | 191 | Load balancer rule names now include the protocol in addition to the LB name and service port. 192 | This was added to distinguish tcp, udp and tcp-proxy services operating on the same port. 193 | Without this change, it would not be possible to map a service that runs on both TCP and UDP port 8000, for example. 194 | 195 | :warning: **If you have existing rules, remove them before the migration, and add them back afterwards.** 196 | 197 | If you don't do this, you will end up with duplicate rules for the same service, which won't work. 198 | 199 | ### Metadata 200 | 201 | Since the controller is now intended to be run inside a pod and not on the node, it will not be able to fetch metadata from the Virtual Router's DHCP server. 202 | 203 | Instead, it first obtains the name of the node from Kubernetes, then fetches information from the CloudStack API. 204 | 205 | ## Development 206 | 207 | ### Building 208 | 209 | At least Go 1.23 is required to build cloudstack-ccm. 210 | 211 | To build the controller with correct versioning, some build flags need to be passed. 212 | A Makefile is provided that sets these build flags to automatically derived values. 213 | 214 | ```bash 215 | go get github.com/apache/cloudstack-kubernetes-provider 216 | cd ${GOPATH}/src/github.com/apache/cloudstack-kubernetes-provider 217 | make 218 | ``` 219 | 220 | To build the cloudstack-cloud-controller-manager container, please use the provided Dockerfile. 221 | The Makefile will also with that and properly tag the resulting container. 222 | 223 | ```bash 224 | make docker 225 | ``` 226 | 227 | ### Testing 228 | 229 | You need a local instance of the CloudStack Management Server or a 'real' one to connect to. 230 | The CCM supports the same cloud-config configuration file format used by [the cs tool](https://github.com/exoscale/cs), 231 | so you can simply point it to that. 232 | 233 | ```bash 234 | ./cloudstack-ccm --cloud-provider external-cloudstack --cloud-config ./cloud-config --kubeconfig ~/.kube/config 235 | ``` 236 | 237 | Replace k8s-apiserver with the host name of your Kubernetes development clusters's API server. 238 | 239 | If you don't have a 'real' CloudStack installation, you can also launch a local [simulator instance](https://hub.docker.com/r/cloudstack/simulator) instead. This is very useful for dry-run testing. 240 | 241 | ### Debugging 242 | 243 | You can use the VSCode extension [Go](https://marketplace.visualstudio.com/items?itemName=golang.go) to debug the CCM. 244 | Add the following configuration to the `.vscode/launch.json` file to launch the CCM and debug it. 245 | 246 | ```json 247 | { 248 | "version": "0.2.0", 249 | "configurations": [ 250 | { 251 | "name": "Launch CloudStack CCM", 252 | "type": "go", 253 | "request": "launch", 254 | "mode": "auto", 255 | "program": "${workspaceFolder}/cmd/cloudstack-ccm", 256 | "env": {}, 257 | "args": [ 258 | "--cloud-provider=external-cloudstack", 259 | "--cloud-config=${workspaceFolder}/cloud-config", 260 | "--kubeconfig=${env:HOME}/.kube/config", 261 | "--leader-elect=false", 262 | "--v=4" 263 | ], 264 | "showLog": true, 265 | "trace": "verbose" 266 | }, 267 | { 268 | "name": "Attach to Process", 269 | "type": "go", 270 | "request": "attach", 271 | "mode": "local", 272 | "processId": 0 273 | } 274 | ] 275 | } 276 | ``` 277 | 278 | ## Copyright 279 | 280 | Copyright 2019 The Apache Software Foundation 281 | 282 | This product includes software developed at 283 | The Apache Software Foundation (http://www.apache.org/). 284 | -------------------------------------------------------------------------------- /cloudstack.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | "context" 24 | "encoding/json" 25 | "errors" 26 | "fmt" 27 | "io" 28 | "os" 29 | "strings" 30 | "time" 31 | 32 | "github.com/apache/cloudstack-go/v2/cloudstack" 33 | "github.com/blang/semver/v4" 34 | "gopkg.in/gcfg.v1" 35 | 36 | corev1 "k8s.io/api/core/v1" 37 | apierrors "k8s.io/apimachinery/pkg/api/errors" 38 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 39 | "k8s.io/apimachinery/pkg/types" 40 | "k8s.io/client-go/kubernetes" 41 | cloudprovider "k8s.io/cloud-provider" 42 | "k8s.io/klog/v2" 43 | ) 44 | 45 | // ProviderName is the name of this cloud provider. 46 | const ProviderName = "external-cloudstack" 47 | 48 | // CSConfig wraps the config for the CloudStack cloud provider. 49 | type CSConfig struct { 50 | Global struct { 51 | APIURL string `gcfg:"api-url"` 52 | APIKey string `gcfg:"api-key"` 53 | SecretKey string `gcfg:"secret-key"` 54 | SSLNoVerify bool `gcfg:"ssl-no-verify"` 55 | ProjectID string `gcfg:"project-id"` 56 | Zone string `gcfg:"zone"` 57 | Region string `gcfg:"region"` 58 | } 59 | } 60 | 61 | // CSCloud is an implementation of Interface for CloudStack. 62 | type CSCloud struct { 63 | client *cloudstack.CloudStackClient 64 | projectID string // If non-"", all resources will be created within this project 65 | zone string 66 | region string 67 | version semver.Version 68 | clientBuilder cloudprovider.ControllerClientBuilder 69 | } 70 | 71 | func init() { 72 | cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { 73 | cfg, err := readConfig(config) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | return newCSCloud(cfg) 79 | }) 80 | } 81 | 82 | func readConfig(config io.Reader) (*CSConfig, error) { 83 | cfg := &CSConfig{} 84 | 85 | if config == nil { 86 | return cfg, nil 87 | } 88 | 89 | if err := gcfg.ReadInto(cfg, config); err != nil { 90 | return nil, fmt.Errorf("could not parse cloud provider config: %v", err) 91 | } 92 | 93 | return cfg, nil 94 | } 95 | 96 | // newCSCloud creates a new instance of CSCloud. 97 | func newCSCloud(cfg *CSConfig) (*CSCloud, error) { 98 | cs := &CSCloud{ 99 | projectID: cfg.Global.ProjectID, 100 | zone: cfg.Global.Zone, 101 | region: cfg.Global.Region, 102 | version: semver.Version{}, 103 | } 104 | 105 | if cfg.Global.APIURL != "" && cfg.Global.APIKey != "" && cfg.Global.SecretKey != "" { 106 | cs.client = cloudstack.NewAsyncClient(cfg.Global.APIURL, cfg.Global.APIKey, cfg.Global.SecretKey, !cfg.Global.SSLNoVerify) 107 | } 108 | 109 | if cs.client == nil { 110 | return nil, errors.New("no cloud provider config given") 111 | } 112 | 113 | version, err := cs.getManagementServerVersion() 114 | if err != nil { 115 | return nil, err 116 | } 117 | cs.version = version 118 | 119 | return cs, nil 120 | } 121 | 122 | func (cs *CSCloud) getManagementServerVersion() (semver.Version, error) { 123 | msServersResp, err := cs.client.Management.ListManagementServersMetrics(cs.client.Management.NewListManagementServersMetricsParams()) 124 | if err != nil { 125 | return semver.Version{}, err 126 | } 127 | if msServersResp.Count == 0 { 128 | return semver.Version{}, errors.New("no management servers found") 129 | } 130 | version := msServersResp.ManagementServersMetrics[0].Version 131 | v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], ".")) 132 | if err != nil { 133 | klog.Errorf("failed to parse management server version: %v", err) 134 | return semver.Version{}, err 135 | } 136 | return v, nil 137 | } 138 | 139 | // Initialize passes a Kubernetes clientBuilder interface to the cloud provider 140 | func (cs *CSCloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { 141 | cs.clientBuilder = clientBuilder 142 | } 143 | 144 | // LoadBalancer returns an implementation of LoadBalancer for CloudStack. 145 | func (cs *CSCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) { 146 | if cs.client == nil { 147 | return nil, false 148 | } 149 | 150 | return cs, true 151 | } 152 | 153 | // Instances returns an implementation of Instances for CloudStack. 154 | func (cs *CSCloud) Instances() (cloudprovider.Instances, bool) { 155 | if cs.client == nil { 156 | return nil, false 157 | } 158 | 159 | return cs, true 160 | } 161 | 162 | func (cs *CSCloud) InstancesV2() (cloudprovider.InstancesV2, bool) { 163 | if cs.client == nil { 164 | return nil, false 165 | } 166 | 167 | return cs, true 168 | } 169 | 170 | // Zones returns an implementation of Zones for CloudStack. 171 | func (cs *CSCloud) Zones() (cloudprovider.Zones, bool) { 172 | if cs.client == nil { 173 | return nil, false 174 | } 175 | 176 | return cs, true 177 | } 178 | 179 | // Clusters returns an implementation of Clusters for CloudStack. 180 | func (cs *CSCloud) Clusters() (cloudprovider.Clusters, bool) { 181 | if cs.client == nil { 182 | return nil, false 183 | } 184 | 185 | klog.Warning("This cloud provider doesn't support clusters") 186 | return nil, false 187 | } 188 | 189 | // Routes returns an implementation of Routes for CloudStack. 190 | func (cs *CSCloud) Routes() (cloudprovider.Routes, bool) { 191 | if cs.client == nil { 192 | return nil, false 193 | } 194 | 195 | klog.Warning("This cloud provider doesn't support routes") 196 | return nil, false 197 | } 198 | 199 | // ProviderName returns the cloud provider ID. 200 | func (cs *CSCloud) ProviderName() string { 201 | return ProviderName 202 | } 203 | 204 | // HasClusterID returns true if the cluster has a clusterID 205 | func (cs *CSCloud) HasClusterID() bool { 206 | return true 207 | } 208 | 209 | // GetZone returns the Zone containing the region that the program is running in. 210 | func (cs *CSCloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) { 211 | zone := cloudprovider.Zone{} 212 | 213 | if cs.zone == "" { 214 | // In Kubernetes pods, os.Hostname() returns the pod name, not the node hostname. 215 | // We need to get the node name from the pod's spec.nodeName using the Kubernetes API. 216 | nodeName, err := cs.getNodeNameFromPod(ctx) 217 | if err != nil { 218 | return zone, fmt.Errorf("failed to get node name for retrieving the zone: %v", err) 219 | } 220 | 221 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName( 222 | nodeName, 223 | cloudstack.WithProject(cs.projectID), 224 | ) 225 | if err != nil { 226 | if count == 0 { 227 | return zone, fmt.Errorf("could not find CloudStack instance with name %s for retrieving the zone: %v", nodeName, err) 228 | } 229 | return zone, fmt.Errorf("error getting instance for retrieving the zone: %v", err) 230 | } 231 | 232 | cs.zone = instance.Zonename 233 | } 234 | 235 | klog.V(2).Infof("Current zone is %v", cs.zone) 236 | zone.FailureDomain = cs.zone 237 | 238 | zone.Region = cs.getRegionFromZone(cs.zone) 239 | 240 | return zone, nil 241 | } 242 | 243 | // GetZoneByProviderID returns the Zone, found by using the provider ID. 244 | func (cs *CSCloud) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { 245 | zone := cloudprovider.Zone{} 246 | 247 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID( 248 | cs.getInstanceIDFromProviderID(providerID), 249 | cloudstack.WithProject(cs.projectID), 250 | ) 251 | if err != nil { 252 | if count == 0 { 253 | return zone, fmt.Errorf("could not find node by ID: %v", providerID) 254 | } 255 | return zone, fmt.Errorf("error retrieving zone: %v", err) 256 | } 257 | 258 | klog.V(2).Infof("Current zone is %v", cs.zone) 259 | zone.FailureDomain = instance.Zonename 260 | zone.Region = cs.getRegionFromZone(instance.Zonename) 261 | 262 | return zone, nil 263 | } 264 | 265 | // GetZoneByNodeName returns the Zone, found by using the node name. 266 | func (cs *CSCloud) GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (cloudprovider.Zone, error) { 267 | zone := cloudprovider.Zone{} 268 | 269 | instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName( 270 | string(nodeName), 271 | cloudstack.WithProject(cs.projectID), 272 | ) 273 | if err != nil { 274 | if count == 0 { 275 | return zone, fmt.Errorf("could not find node: %v", nodeName) 276 | } 277 | return zone, fmt.Errorf("error retrieving zone: %v", err) 278 | } 279 | 280 | klog.V(2).Infof("Current zone is %v", cs.zone) 281 | zone.FailureDomain = instance.Zonename 282 | zone.Region = cs.getRegionFromZone(instance.Zonename) 283 | 284 | return zone, nil 285 | } 286 | 287 | // getNodeNameFromPod gets the node name where this pod is running by querying the Kubernetes API. 288 | // It uses the pod's name and namespace (from environment variables or hostname) to look up the pod 289 | // and retrieve its spec.nodeName field. 290 | func (cs *CSCloud) getNodeNameFromPod(ctx context.Context) (string, error) { 291 | if cs.clientBuilder == nil { 292 | return "", fmt.Errorf("clientBuilder not initialized, cannot query Kubernetes API") 293 | } 294 | 295 | client, err := cs.clientBuilder.Client("cloud-controller-manager") 296 | if err != nil { 297 | return "", fmt.Errorf("failed to get Kubernetes client: %v", err) 298 | } 299 | 300 | // Get pod name and namespace 301 | // In Kubernetes, the pod name is available as HOSTNAME environment variable 302 | // or we can use os.Hostname() which returns the pod name 303 | podName := os.Getenv("HOSTNAME") 304 | if podName == "" { 305 | var err error 306 | podName, err = os.Hostname() 307 | if err != nil { 308 | return "", fmt.Errorf("failed to get pod name: %v", err) 309 | } 310 | } 311 | 312 | // Get namespace from environment variable or default to kube-system for CCM 313 | namespace := os.Getenv("POD_NAMESPACE") 314 | if namespace == "" { 315 | // Try reading from service account namespace file (available in pods) 316 | if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { 317 | namespace = string(data) 318 | } else { 319 | // Default namespace for cloud controller manager 320 | namespace = "kube-system" 321 | } 322 | } 323 | 324 | // Get the pod object from Kubernetes API 325 | pod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) 326 | if err != nil { 327 | return "", fmt.Errorf("failed to get pod %s/%s from Kubernetes API: %v", namespace, podName, err) 328 | } 329 | 330 | if pod.Spec.NodeName == "" { 331 | return "", fmt.Errorf("pod %s/%s does not have a nodeName assigned yet", namespace, podName) 332 | } 333 | 334 | klog.V(4).Infof("found node name %s for pod %s/%s", pod.Spec.NodeName, namespace, podName) 335 | return pod.Spec.NodeName, nil 336 | } 337 | 338 | // setServiceAnnotation updates a service annotation using the Kubernetes client. 339 | // It uses a patch operation with retry logic to handle concurrent updates safely. 340 | func (cs *CSCloud) setServiceAnnotation(ctx context.Context, service *corev1.Service, key, value string) error { 341 | if cs.clientBuilder == nil { 342 | klog.V(4).Infof("Client builder not available, skipping annotation update for service %s/%s", service.Namespace, service.Name) 343 | return nil 344 | } 345 | 346 | client, err := cs.clientBuilder.Client("cloud-controller-manager") 347 | if err != nil { 348 | return fmt.Errorf("failed to get Kubernetes client: %v", err) 349 | } 350 | 351 | // First, check if the annotation already has the correct value to avoid unnecessary updates 352 | svc, err := client.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{}) 353 | if err != nil { 354 | if apierrors.IsNotFound(err) { 355 | klog.V(4).Infof("Service %s/%s not found, skipping annotation update", service.Namespace, service.Name) 356 | return nil 357 | } 358 | return fmt.Errorf("failed to get service: %v", err) 359 | } 360 | 361 | // Check if annotation already has the correct value 362 | if svc.Annotations != nil { 363 | if existingValue, exists := svc.Annotations[key]; exists && existingValue == value { 364 | klog.V(4).Infof("Annotation %s already set to %s for service %s/%s", key, value, service.Namespace, service.Name) 365 | return nil 366 | } 367 | } 368 | 369 | // Use patch operation with retry logic to handle concurrent updates 370 | return cs.patchServiceAnnotation(ctx, client, service.Namespace, service.Name, key, value) 371 | } 372 | 373 | // patchServiceAnnotation patches a service annotation using a JSON merge patch with retry logic. 374 | // This method handles concurrent updates safely by retrying on conflicts. 375 | func (cs *CSCloud) patchServiceAnnotation(ctx context.Context, client kubernetes.Interface, namespace, name, key, value string) error { 376 | const maxRetries = 3 377 | const retryDelay = 500 * time.Millisecond 378 | 379 | // Prepare the patch payload - merge patch that updates only the specific annotation 380 | // JSON merge patch will preserve other annotations while updating/adding this one 381 | patchData := map[string]interface{}{ 382 | "metadata": map[string]interface{}{ 383 | "annotations": map[string]string{ 384 | key: value, 385 | }, 386 | }, 387 | } 388 | 389 | patchBytes, err := json.Marshal(patchData) 390 | if err != nil { 391 | return fmt.Errorf("failed to marshal patch data: %v", err) 392 | } 393 | 394 | for attempt := 0; attempt < maxRetries; attempt++ { 395 | // Apply the patch using JSON merge patch type 396 | // This is atomic and avoids race conditions by merging with existing annotations 397 | _, err = client.CoreV1().Services(namespace).Patch( 398 | ctx, 399 | name, 400 | types.MergePatchType, 401 | patchBytes, 402 | metav1.PatchOptions{}, 403 | ) 404 | 405 | if err == nil { 406 | klog.V(4).Infof("Successfully set annotation %s=%s on service %s/%s", key, value, namespace, name) 407 | return nil 408 | } 409 | 410 | // Handle conflict errors with retry logic 411 | if apierrors.IsConflict(err) { 412 | if attempt < maxRetries-1 { 413 | klog.V(4).Infof("Conflict updating service %s/%s annotation, retrying (attempt %d/%d): %v", namespace, name, attempt+1, maxRetries, err) 414 | time.Sleep(retryDelay) 415 | continue 416 | } 417 | return fmt.Errorf("failed to update service annotation after %d retries due to conflicts: %v", maxRetries, err) 418 | } 419 | 420 | // Handle not found errors 421 | if apierrors.IsNotFound(err) { 422 | klog.V(4).Infof("Service %s/%s not found during patch, skipping annotation update", namespace, name) 423 | return nil 424 | } 425 | 426 | // For other errors, return immediately 427 | return fmt.Errorf("failed to patch service annotation: %v", err) 428 | } 429 | 430 | return fmt.Errorf("failed to update service annotation after %d attempts", maxRetries) 431 | } 432 | 433 | func (cs *CSCloud) getRegionFromZone(zone string) string { 434 | if cs.region != "" { 435 | return cs.region 436 | } 437 | return zone 438 | } 439 | -------------------------------------------------------------------------------- /cloudstack_loadbalancer.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cloudstack 21 | 22 | import ( 23 | "context" 24 | "fmt" 25 | "net" 26 | "strconv" 27 | "strings" 28 | 29 | "github.com/apache/cloudstack-go/v2/cloudstack" 30 | "github.com/blang/semver/v4" 31 | "k8s.io/klog/v2" 32 | 33 | corev1 "k8s.io/api/core/v1" 34 | cloudprovider "k8s.io/cloud-provider" 35 | ) 36 | 37 | const ( 38 | // defaultAllowedCIDR is the network range that is allowed on the firewall 39 | // by default when no explicit CIDR list is given on a LoadBalancer. 40 | defaultAllowedCIDR = "0.0.0.0/0" 41 | 42 | // ServiceAnnotationLoadBalancerProxyProtocol is the annotation used on the 43 | // service to enable the proxy protocol on a CloudStack load balancer. 44 | // Note that this protocol only applies to TCP service ports and 45 | // CloudStack >= 4.6 is required for it to work. 46 | ServiceAnnotationLoadBalancerProxyProtocol = "service.beta.kubernetes.io/cloudstack-load-balancer-proxy-protocol" 47 | ServiceAnnotationLoadBalancerLoadbalancerHostname = "service.beta.kubernetes.io/cloudstack-load-balancer-hostname" 48 | 49 | // ServiceAnnotationLoadBalancerSourceCidrs is the annotation used on the 50 | // service to specify the source CIDR list for a CloudStack load balancer. 51 | // The CIDR list is a comma-separated list of CIDR ranges (e.g., "10.0.0.0/8,192.168.1.0/24"). 52 | // If not specified, the default is to allow all sources ("0.0.0.0/0"). 53 | ServiceAnnotationLoadBalancerSourceCidrs = "service.beta.kubernetes.io/cloudstack-load-balancer-source-cidrs" 54 | 55 | // ServiceAnnotationLoadBalancerIPAssociatedByController indicates that the controller 56 | // associated the IP address. This annotation is set by the controller when it associates 57 | // an unallocated IP, and is used to determine if the IP should be disassociated on deletion. 58 | ServiceAnnotationLoadBalancerIPAssociatedByController = "service.beta.kubernetes.io/cloudstack-load-balancer-ip-associated-by-controller" //nolint:gosec 59 | ) 60 | 61 | type loadBalancer struct { 62 | *cloudstack.CloudStackClient 63 | 64 | name string 65 | algorithm string 66 | hostIDs []string 67 | ipAddr string 68 | ipAddrID string 69 | networkID string 70 | projectID string 71 | rules map[string]*cloudstack.LoadBalancerRule 72 | ipAssociatedByController bool 73 | } 74 | 75 | // GetLoadBalancer returns whether the specified load balancer exists, and if so, what its status is. 76 | func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service) (*corev1.LoadBalancerStatus, bool, error) { 77 | klog.V(4).Infof("GetLoadBalancer(%v, %v, %v)", clusterName, service.Namespace, service.Name) 78 | 79 | // Get the load balancer details and existing rules. 80 | lb, err := cs.getLoadBalancer(service) 81 | if err != nil { 82 | return nil, false, err 83 | } 84 | 85 | // If we don't have any rules, the load balancer does not exist. 86 | if len(lb.rules) == 0 { 87 | return nil, false, nil 88 | } 89 | 90 | klog.V(4).Infof("Found a load balancer associated with IP %v", lb.ipAddr) 91 | 92 | status := &corev1.LoadBalancerStatus{} 93 | status.Ingress = append(status.Ingress, corev1.LoadBalancerIngress{IP: lb.ipAddr}) 94 | 95 | return status, true, nil 96 | } 97 | 98 | // EnsureLoadBalancer creates a new load balancer, or updates the existing one. Returns the status of the balancer. 99 | func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) (status *corev1.LoadBalancerStatus, err error) { 100 | klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, service.Spec.LoadBalancerIP, service.Spec.Ports, nodes) 101 | 102 | if len(service.Spec.Ports) == 0 { 103 | return nil, fmt.Errorf("requested load balancer with no ports") 104 | } 105 | 106 | // Get the load balancer details and existing rules. 107 | lb, err := cs.getLoadBalancer(service) 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | // Set the load balancer algorithm. 113 | switch service.Spec.SessionAffinity { 114 | case corev1.ServiceAffinityNone: 115 | lb.algorithm = "roundrobin" 116 | case corev1.ServiceAffinityClientIP: 117 | lb.algorithm = "source" 118 | default: 119 | return nil, fmt.Errorf("unsupported load balancer affinity: %v", service.Spec.SessionAffinity) 120 | } 121 | 122 | // Verify that all the hosts belong to the same network, and retrieve their ID's. 123 | lb.hostIDs, lb.networkID, err = cs.verifyHosts(nodes) 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | if !lb.hasLoadBalancerIP() { 129 | // Create or retrieve the load balancer IP. 130 | if err := lb.getLoadBalancerIP(service.Spec.LoadBalancerIP); err != nil { 131 | return nil, err 132 | } 133 | 134 | if lb.ipAddr != "" && lb.ipAddr != service.Spec.LoadBalancerIP { 135 | defer func(lb *loadBalancer) { 136 | if err != nil { 137 | if err := lb.releaseLoadBalancerIP(); err != nil { 138 | klog.Errorf(err.Error()) 139 | } 140 | } 141 | }(lb) 142 | } 143 | 144 | // If the controller associated the IP and matches the service spec, set the annotation to persist this information. 145 | if lb.ipAssociatedByController && lb.ipAddr == service.Spec.LoadBalancerIP { 146 | if err := cs.setServiceAnnotation(ctx, service, ServiceAnnotationLoadBalancerIPAssociatedByController, "true"); err != nil { 147 | // Log the error but don't fail - the annotation is helpful but not critical 148 | klog.Warningf("Failed to set annotation on service %s/%s: %v", service.Namespace, service.Name, err) 149 | } 150 | } 151 | } 152 | 153 | klog.V(4).Infof("Load balancer %v is associated with IP %v", lb.name, lb.ipAddr) 154 | 155 | for _, port := range service.Spec.Ports { 156 | // Construct the protocol name first, we need it a few times 157 | protocol := ProtocolFromServicePort(port, service) 158 | if protocol == LoadBalancerProtocolInvalid { 159 | return nil, fmt.Errorf("unsupported load balancer protocol: %v", port.Protocol) 160 | } 161 | 162 | // All ports have their own load balancer rule, so add the port to lbName to keep the names unique. 163 | lbRuleName := fmt.Sprintf("%s-%s-%d", lb.name, protocol, port.Port) 164 | 165 | // If the load balancer rule exists and is up-to-date, we move on to the next rule. 166 | lbRule, needsUpdate, err := lb.checkLoadBalancerRule(lbRuleName, port, protocol, service, cs.version) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | if lbRule != nil { 172 | if needsUpdate { 173 | klog.V(4).Infof("Updating load balancer rule: %v", lbRuleName) 174 | if err := lb.updateLoadBalancerRule(lbRuleName, protocol, service, cs.version); err != nil { 175 | return nil, err 176 | } 177 | // Delete the rule from the map, to prevent it being deleted. 178 | delete(lb.rules, lbRuleName) 179 | } else { 180 | klog.V(4).Infof("Load balancer rule %v is up-to-date", lbRuleName) 181 | // Delete the rule from the map, to prevent it being deleted. 182 | delete(lb.rules, lbRuleName) 183 | } 184 | } else { 185 | klog.V(4).Infof("Creating load balancer rule: %v", lbRuleName) 186 | lbRule, err = lb.createLoadBalancerRule(lbRuleName, port, protocol, service) 187 | if err != nil { 188 | return nil, err 189 | } 190 | 191 | klog.V(4).Infof("Assigning hosts (%v) to load balancer rule: %v", lb.hostIDs, lbRuleName) 192 | if err = lb.assignHostsToRule(lbRule, lb.hostIDs); err != nil { 193 | return nil, err 194 | } 195 | } 196 | 197 | network, count, err := lb.Network.GetNetworkByID(lb.networkID, cloudstack.WithProject(lb.projectID)) 198 | if err != nil { 199 | if count == 0 { 200 | return nil, err 201 | } 202 | return nil, err 203 | } 204 | 205 | if lbRule != nil { 206 | if isFirewallSupported(network.Service) { 207 | klog.V(4).Infof("Creating firewall rules for load balancer rule: %v (%v:%v:%v)", lbRuleName, protocol, lbRule.Publicip, port.Port) 208 | if _, err := lb.updateFirewallRule(lbRule.Publicipid, int(port.Port), protocol, service.Spec.LoadBalancerSourceRanges); err != nil { 209 | return nil, err 210 | } 211 | } else if isNetworkACLSupported(network.Service) { 212 | klog.V(4).Infof("Creating ACL rules for load balancer rule: %v (%v:%v:%v)", lbRuleName, protocol, lbRule.Publicip, port.Port) 213 | if _, err := lb.updateNetworkACL(int(port.Port), protocol, network.Id); err != nil { 214 | return nil, err 215 | } 216 | } 217 | } 218 | } 219 | 220 | // Cleanup any rules that are now still in the rules map, as they are no longer needed. 221 | for _, lbRule := range lb.rules { 222 | protocol := ProtocolFromLoadBalancer(lbRule.Protocol) 223 | if protocol == LoadBalancerProtocolInvalid { 224 | return nil, fmt.Errorf("error parsing protocol %v: %v", lbRule.Protocol, err) 225 | } 226 | port, err := strconv.ParseInt(lbRule.Publicport, 10, 32) 227 | if err != nil { 228 | return nil, fmt.Errorf("error parsing port %s: %v", lbRule.Publicport, err) 229 | } 230 | 231 | klog.V(4).Infof("Deleting firewall rules associated with load balancer rule: %v (%v:%v:%v)", lbRule.Name, protocol, lbRule.Publicip, port) 232 | if _, err := lb.deleteFirewallRule(lbRule.Publicipid, int(port), protocol); err != nil { 233 | return nil, err 234 | } 235 | 236 | klog.V(4).Infof("Deleting Network ACL rules associated with load balancer rule: %v (%v:%v)", lbRule.Name, protocol, port) 237 | if _, err := lb.deleteNetworkACLRule(int(port), protocol, lb.networkID); err != nil { 238 | return nil, err 239 | } 240 | 241 | klog.V(4).Infof("Deleting obsolete load balancer rule: %v", lbRule.Name) 242 | if err := lb.deleteLoadBalancerRule(lbRule); err != nil { 243 | return nil, err 244 | } 245 | } 246 | 247 | status = &corev1.LoadBalancerStatus{} 248 | // If hostname is explicitly set using service annotation 249 | // Workaround for https://github.com/kubernetes/kubernetes/issues/66607 250 | if hostname := getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerLoadbalancerHostname, ""); hostname != "" { 251 | status.Ingress = []corev1.LoadBalancerIngress{{Hostname: hostname}} 252 | return status, nil 253 | } 254 | // Default to IP 255 | status.Ingress = []corev1.LoadBalancerIngress{{IP: lb.ipAddr}} 256 | 257 | return status, nil 258 | } 259 | 260 | // UpdateLoadBalancer updates hosts under the specified load balancer. 261 | func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error { 262 | klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v)", clusterName, service.Namespace, service.Name, nodes) 263 | 264 | // Get the load balancer details and existing rules. 265 | lb, err := cs.getLoadBalancer(service) 266 | if err != nil { 267 | return err 268 | } 269 | 270 | // Verify that all the hosts belong to the same network, and retrieve their ID's. 271 | lb.hostIDs, _, err = cs.verifyHosts(nodes) 272 | if err != nil { 273 | return err 274 | } 275 | 276 | for _, lbRule := range lb.rules { 277 | p := lb.LoadBalancer.NewListLoadBalancerRuleInstancesParams(lbRule.Id) 278 | 279 | // Retrieve all VMs currently associated to this load balancer rule. 280 | l, err := lb.LoadBalancer.ListLoadBalancerRuleInstances(p) 281 | if err != nil { 282 | return fmt.Errorf("error retrieving associated instances: %v", err) 283 | } 284 | 285 | assign, remove := symmetricDifference(lb.hostIDs, l.LoadBalancerRuleInstances) 286 | 287 | if len(assign) > 0 { 288 | klog.V(4).Infof("Assigning new hosts (%v) to load balancer rule: %v", assign, lbRule.Name) 289 | if err := lb.assignHostsToRule(lbRule, assign); err != nil { 290 | return err 291 | } 292 | } 293 | 294 | if len(remove) > 0 { 295 | klog.V(4).Infof("Removing old hosts (%v) from load balancer rule: %v", assign, lbRule.Name) 296 | if err := lb.removeHostsFromRule(lbRule, remove); err != nil { 297 | return err 298 | } 299 | } 300 | } 301 | 302 | return nil 303 | } 304 | 305 | func isFirewallSupported(services []cloudstack.NetworkServiceInternal) bool { 306 | for _, svc := range services { 307 | if svc.Name == "Firewall" { 308 | return true 309 | } 310 | } 311 | return false 312 | } 313 | 314 | func isNetworkACLSupported(services []cloudstack.NetworkServiceInternal) bool { 315 | for _, svc := range services { 316 | if svc.Name == "NetworkACL" { 317 | return true 318 | } 319 | } 320 | return false 321 | } 322 | 323 | // EnsureLoadBalancerDeleted deletes the specified load balancer if it exists, returning 324 | // nil if the load balancer specified either didn't exist or was successfully deleted. 325 | func (cs *CSCloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *corev1.Service) error { 326 | klog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v)", clusterName, service.Namespace, service.Name) 327 | 328 | // Get the load balancer details and existing rules. 329 | lb, err := cs.getLoadBalancer(service) 330 | if err != nil { 331 | return err 332 | } 333 | 334 | for _, lbRule := range lb.rules { 335 | klog.V(4).Infof("Deleting firewall rules / Network ACLs for load balancer: %v", lbRule.Name) 336 | protocol := ProtocolFromLoadBalancer(lbRule.Protocol) 337 | if protocol == LoadBalancerProtocolInvalid { 338 | klog.Errorf("Error parsing protocol: %v", lbRule.Protocol) 339 | } else { 340 | port, err := strconv.ParseInt(lbRule.Publicport, 10, 32) 341 | if err != nil { 342 | klog.Errorf("Error parsing port: %v", err) 343 | } else { 344 | networkId, err := cs.getNetworkIDFromIPAddress(lb.ipAddrID) 345 | if err != nil { 346 | return err 347 | } 348 | network, count, err := lb.Network.GetNetworkByID(networkId, cloudstack.WithProject(lb.projectID)) 349 | if err != nil { 350 | if count == 0 { 351 | klog.Errorf("No network found with ID: %v", networkId) 352 | return err 353 | } 354 | return err 355 | } 356 | if network.Vpcid == "" { 357 | _, err = lb.deleteFirewallRule(lbRule.Publicipid, int(port), protocol) 358 | if err != nil { 359 | klog.Errorf("Error deleting firewall rule: %v", err) 360 | } 361 | } else { 362 | klog.V(4).Infof("Deleting network ACLs for %v - %v", int(port), protocol) 363 | _, err = lb.deleteNetworkACLRule(int(port), protocol, networkId) 364 | if err != nil { 365 | klog.Errorf("Error deleting Network ACL rule: %v", err) 366 | } 367 | } 368 | } 369 | 370 | klog.V(4).Infof("Deleting load balancer rule: %v", lbRule.Name) 371 | if err := lb.deleteLoadBalancerRule(lbRule); err != nil { 372 | return err 373 | } 374 | } 375 | } 376 | 377 | if lb.ipAddr != "" { 378 | // If the IP was allocated by the controller (not specified in service spec), release it. 379 | if lb.ipAddr != service.Spec.LoadBalancerIP { 380 | klog.V(4).Infof("Releasing load balancer IP: %v", lb.ipAddr) 381 | if err := lb.releaseLoadBalancerIP(); err != nil { 382 | return err 383 | } 384 | } else { 385 | // If the IP was specified in service spec, check if it was associated by the controller. 386 | // First, check if there's an annotation indicating the controller associated it. 387 | // If not, check if there are any other load balancer rules using this IP. 388 | shouldDisassociate := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerIPAssociatedByController, false) 389 | 390 | if shouldDisassociate { 391 | // Annotation is set, so check if there are any other load balancer rules using this IP. 392 | // Since we've already deleted all rules for this service, any remaining rules must belong 393 | // to other services. If no other rules exist, it's safe to disassociate the IP. 394 | ip, count, err := lb.Address.GetPublicIpAddressByID(lb.ipAddrID) 395 | if err != nil { 396 | klog.Errorf("Error retrieving IP address %v for disassociation check: %v", lb.ipAddr, err) 397 | shouldDisassociate = false 398 | } else if count > 0 && ip.Allocated != "" { 399 | p := lb.LoadBalancer.NewListLoadBalancerRulesParams() 400 | p.SetPublicipid(lb.ipAddrID) 401 | p.SetListall(true) 402 | if lb.projectID != "" { 403 | p.SetProjectid(lb.projectID) 404 | } 405 | otherRules, err := lb.LoadBalancer.ListLoadBalancerRules(p) 406 | if err != nil { 407 | klog.Errorf("Error checking for other load balancer rules using IP %v: %v", lb.ipAddr, err) 408 | shouldDisassociate = false 409 | } else if otherRules.Count > 0 { 410 | // Other load balancer rules are using this IP (other services are using it), 411 | // so don't disassociate. 412 | shouldDisassociate = false 413 | } 414 | } 415 | } 416 | 417 | if shouldDisassociate { 418 | klog.V(4).Infof("Disassociating IP %v that was associated by the controller", lb.ipAddr) 419 | if err := lb.releaseLoadBalancerIP(); err != nil { 420 | return err 421 | } 422 | } 423 | } 424 | } 425 | 426 | return nil 427 | } 428 | 429 | // GetLoadBalancerName retrieves the name of the LoadBalancer. 430 | func (cs *CSCloud) GetLoadBalancerName(ctx context.Context, clusterName string, service *corev1.Service) string { 431 | return cloudprovider.DefaultLoadBalancerName(service) 432 | } 433 | 434 | // getLoadBalancer retrieves the IP address and ID and all the existing rules it can find. 435 | func (cs *CSCloud) getLoadBalancer(service *corev1.Service) (*loadBalancer, error) { 436 | lb := &loadBalancer{ 437 | CloudStackClient: cs.client, 438 | name: cs.GetLoadBalancerName(context.TODO(), "", service), 439 | projectID: cs.projectID, 440 | rules: make(map[string]*cloudstack.LoadBalancerRule), 441 | } 442 | 443 | p := cs.client.LoadBalancer.NewListLoadBalancerRulesParams() 444 | p.SetKeyword(lb.name) 445 | p.SetListall(true) 446 | 447 | if cs.projectID != "" { 448 | p.SetProjectid(cs.projectID) 449 | } 450 | 451 | l, err := cs.client.LoadBalancer.ListLoadBalancerRules(p) 452 | if err != nil { 453 | return nil, fmt.Errorf("error retrieving load balancer rules: %v", err) 454 | } 455 | 456 | for _, lbRule := range l.LoadBalancerRules { 457 | lb.rules[lbRule.Name] = lbRule 458 | 459 | if lb.ipAddr != "" && lb.ipAddr != lbRule.Publicip { 460 | klog.Warningf("Load balancer for service %v/%v has rules associated with different IP's: %v, %v", service.Namespace, service.Name, lb.ipAddr, lbRule.Publicip) 461 | } 462 | 463 | lb.ipAddr = lbRule.Publicip 464 | lb.ipAddrID = lbRule.Publicipid 465 | } 466 | 467 | klog.V(4).Infof("Load balancer %v contains %d rule(s)", lb.name, len(lb.rules)) 468 | 469 | return lb, nil 470 | } 471 | 472 | // Get network ID from Public IP Address 473 | func (cs *CSCloud) getNetworkIDFromIPAddress(publicIpId string) (string, error) { 474 | ip, count, err := cs.client.Address.GetPublicIpAddressByID(publicIpId) 475 | if err != nil { 476 | klog.Errorf("Failed to fetch the public IP for id: %v", publicIpId) 477 | return "", err 478 | } 479 | if count == 0 { 480 | return "", err 481 | } 482 | if ip.Networkid != "" { 483 | network, _, netErr := cs.client.Network.GetNetworkByID(ip.Associatednetworkid) 484 | if netErr != nil { 485 | klog.Errorf("Failed to fetch the network for id: %v", ip.Associatednetworkid) 486 | return "", err 487 | } 488 | return network.Id, nil 489 | } 490 | return "", nil 491 | } 492 | 493 | // verifyHosts verifies if all hosts belong to the same network, and returns the host ID's and network ID. 494 | func (cs *CSCloud) verifyHosts(nodes []*corev1.Node) ([]string, string, error) { 495 | hostNames := map[string]bool{} 496 | for _, node := range nodes { 497 | // node.Name can be an FQDN as well, and CloudStack VM names aren't 498 | // To match, we need to Split the domain part off here, if present 499 | hostNames[strings.Split(strings.ToLower(node.Name), ".")[0]] = true 500 | } 501 | 502 | p := cs.client.VirtualMachine.NewListVirtualMachinesParams() 503 | p.SetListall(true) 504 | p.SetDetails([]string{"min", "nics"}) 505 | 506 | if cs.projectID != "" { 507 | p.SetProjectid(cs.projectID) 508 | } 509 | 510 | l, err := cs.client.VirtualMachine.ListVirtualMachines(p) 511 | if err != nil { 512 | return nil, "", fmt.Errorf("error retrieving list of hosts: %v", err) 513 | } 514 | 515 | var hostIDs []string 516 | var networkID string 517 | 518 | // Check if the virtual machine is in the hosts slice, then add the corresponding ID. 519 | for _, vm := range l.VirtualMachines { 520 | if hostNames[strings.ToLower(vm.Name)] { 521 | if networkID != "" && networkID != vm.Nic[0].Networkid { 522 | return nil, "", fmt.Errorf("found hosts that belong to different networks") 523 | } 524 | 525 | networkID = vm.Nic[0].Networkid 526 | hostIDs = append(hostIDs, vm.Id) 527 | } 528 | } 529 | 530 | if len(hostIDs) == 0 || len(networkID) == 0 { 531 | return nil, "", fmt.Errorf("none of the hosts matched the list of VMs retrieved from CS API") 532 | } 533 | 534 | return hostIDs, networkID, nil 535 | } 536 | 537 | // hasLoadBalancerIP returns true if we have a load balancer address and ID. 538 | func (lb *loadBalancer) hasLoadBalancerIP() bool { 539 | return lb.ipAddr != "" && lb.ipAddrID != "" 540 | } 541 | 542 | // getLoadBalancerIP retrieves an existing IP or associates a new IP. 543 | func (lb *loadBalancer) getLoadBalancerIP(loadBalancerIP string) error { 544 | if loadBalancerIP != "" { 545 | return lb.getPublicIPAddress(loadBalancerIP) 546 | } 547 | 548 | return lb.associatePublicIPAddress() 549 | } 550 | 551 | // getPublicIPAddressID retrieves the ID of the given IP, and sets the address and it's ID. 552 | func (lb *loadBalancer) getPublicIPAddress(loadBalancerIP string) error { 553 | klog.V(4).Infof("Retrieve load balancer IP details: %v", loadBalancerIP) 554 | 555 | p := lb.Address.NewListPublicIpAddressesParams() 556 | p.SetIpaddress(loadBalancerIP) 557 | p.SetAllocatedonly(false) 558 | p.SetListall(true) 559 | 560 | if lb.projectID != "" { 561 | p.SetProjectid(lb.projectID) 562 | } 563 | 564 | l, err := lb.Address.ListPublicIpAddresses(p) 565 | if err != nil { 566 | return fmt.Errorf("error retrieving IP address: %v", err) 567 | } 568 | 569 | if l.Count != 1 { 570 | return fmt.Errorf("could not find IP address %v. Found %d addresses", loadBalancerIP, l.Count) 571 | } 572 | 573 | lb.ipAddr = l.PublicIpAddresses[0].Ipaddress 574 | lb.ipAddrID = l.PublicIpAddresses[0].Id 575 | 576 | // If the IP is not allocated, associate it. 577 | if l.PublicIpAddresses[0].Allocated == "" { 578 | return lb.associatePublicIPAddress() 579 | } 580 | return nil 581 | } 582 | 583 | // associatePublicIPAddress associates a new IP and sets the address and it's ID. 584 | func (lb *loadBalancer) associatePublicIPAddress() error { 585 | klog.V(4).Infof("Allocate new IP for load balancer: %v", lb.name) 586 | // If a network belongs to a VPC, the IP address needs to be associated with 587 | // the VPC instead of with the network. 588 | network, count, err := lb.Network.GetNetworkByID(lb.networkID, cloudstack.WithProject(lb.projectID)) 589 | if err != nil { 590 | if count == 0 { 591 | return fmt.Errorf("could not find network %v", lb.networkID) 592 | } 593 | return fmt.Errorf("error retrieving network: %v", err) 594 | } 595 | 596 | p := lb.Address.NewAssociateIpAddressParams() 597 | 598 | if network.Vpcid != "" { 599 | p.SetVpcid(network.Vpcid) 600 | } else { 601 | p.SetNetworkid(lb.networkID) 602 | } 603 | 604 | if lb.projectID != "" { 605 | p.SetProjectid(lb.projectID) 606 | } 607 | 608 | if lb.ipAddr != "" { 609 | p.SetIpaddress(lb.ipAddr) 610 | } 611 | 612 | // Associate a new IP address 613 | r, err := lb.Address.AssociateIpAddress(p) 614 | if err != nil { 615 | return fmt.Errorf("error associating new IP address: %v", err) 616 | } 617 | 618 | lb.ipAddr = r.Ipaddress 619 | lb.ipAddrID = r.Id 620 | lb.ipAssociatedByController = true 621 | 622 | return nil 623 | } 624 | 625 | // releasePublicIPAddress releases an associated IP. 626 | func (lb *loadBalancer) releaseLoadBalancerIP() error { 627 | p := lb.Address.NewDisassociateIpAddressParams(lb.ipAddrID) 628 | 629 | if _, err := lb.Address.DisassociateIpAddress(p); err != nil { 630 | return fmt.Errorf("error releasing load balancer IP %v: %v", lb.ipAddr, err) 631 | } 632 | 633 | return nil 634 | } 635 | 636 | func (lb *loadBalancer) getCIDRList(service *corev1.Service) ([]string, error) { 637 | sourceCIDRs := getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSourceCidrs, defaultAllowedCIDR) 638 | var cidrList []string 639 | if sourceCIDRs != "" { 640 | cidrList = strings.Split(sourceCIDRs, ",") 641 | for i, cidr := range cidrList { 642 | cidr = strings.TrimSpace(cidr) 643 | if _, _, err := net.ParseCIDR(cidr); err != nil { 644 | return nil, fmt.Errorf("invalid CIDR %s in annotation %s: %w", cidr, ServiceAnnotationLoadBalancerSourceCidrs, err) 645 | } 646 | cidrList[i] = cidr 647 | } 648 | } 649 | return cidrList, nil 650 | } 651 | 652 | // checkLoadBalancerRule checks if the rule already exists and if it does, if it can be updated. If 653 | // it does exist but cannot be updated, it will delete the existing rule so it can be created again. 654 | func (lb *loadBalancer) checkLoadBalancerRule(lbRuleName string, port corev1.ServicePort, protocol LoadBalancerProtocol, service *corev1.Service, version semver.Version) (*cloudstack.LoadBalancerRule, bool, error) { 655 | lbRule, ok := lb.rules[lbRuleName] 656 | if !ok { 657 | return nil, false, nil 658 | } 659 | 660 | cidrList, err := lb.getCIDRList(service) 661 | if err != nil { 662 | return nil, false, err 663 | } 664 | 665 | var lbRuleCidrList []string 666 | if lbRule.Cidrlist != "" { 667 | lbRuleCidrList = strings.Split(lbRule.Cidrlist, " ") 668 | for i, cidr := range lbRuleCidrList { 669 | cidr = strings.TrimSpace(cidr) 670 | lbRuleCidrList[i] = cidr 671 | } 672 | } 673 | 674 | // Check if basic properties match (IP and ports). If not, we need to recreate the rule. 675 | basicPropsMatch := lbRule.Publicip == lb.ipAddr && 676 | lbRule.Privateport == strconv.Itoa(int(port.NodePort)) && 677 | lbRule.Publicport == strconv.Itoa(int(port.Port)) 678 | 679 | cidrListChanged := len(cidrList) != len(lbRuleCidrList) || !compareStringSlice(cidrList, lbRuleCidrList) 680 | 681 | // Check if CIDR list also changed and version < 4.22, then we must recreate the rule. 682 | if !basicPropsMatch || (cidrListChanged && version.LT(semver.Version{Major: 4, Minor: 22, Patch: 0})) { 683 | // Delete the load balancer rule so we can create a new one using the new values. 684 | if err := lb.deleteLoadBalancerRule(lbRule); err != nil { 685 | return nil, false, err 686 | } 687 | return nil, false, nil 688 | } 689 | 690 | // Rule can be updated. Check what needs updating. 691 | updateAlgo := lbRule.Algorithm != lb.algorithm 692 | updateProto := lbRule.Protocol != protocol.CSProtocol() 693 | 694 | return lbRule, updateAlgo || updateProto || cidrListChanged, nil 695 | } 696 | 697 | // updateLoadBalancerRule updates a load balancer rule. 698 | func (lb *loadBalancer) updateLoadBalancerRule(lbRuleName string, protocol LoadBalancerProtocol, service *corev1.Service, version semver.Version) error { 699 | lbRule := lb.rules[lbRuleName] 700 | 701 | p := lb.LoadBalancer.NewUpdateLoadBalancerRuleParams(lbRule.Id) 702 | p.SetAlgorithm(lb.algorithm) 703 | p.SetProtocol(protocol.CSProtocol()) 704 | 705 | // If version >= 4.22, we can update the CIDR list. 706 | if version.GTE(semver.Version{Major: 4, Minor: 22, Patch: 0}) { 707 | cidrList, err := lb.getCIDRList(service) 708 | if err != nil { 709 | return err 710 | } 711 | p.SetCidrlist(cidrList) 712 | } 713 | 714 | _, err := lb.LoadBalancer.UpdateLoadBalancerRule(p) 715 | return err 716 | } 717 | 718 | // createLoadBalancerRule creates a new load balancer rule and returns it's ID. 719 | func (lb *loadBalancer) createLoadBalancerRule(lbRuleName string, port corev1.ServicePort, protocol LoadBalancerProtocol, service *corev1.Service) (*cloudstack.LoadBalancerRule, error) { 720 | p := lb.LoadBalancer.NewCreateLoadBalancerRuleParams( 721 | lb.algorithm, 722 | lbRuleName, 723 | int(port.NodePort), 724 | int(port.Port), 725 | ) 726 | 727 | p.SetNetworkid(lb.networkID) 728 | p.SetPublicipid(lb.ipAddrID) 729 | p.SetProtocol(protocol.CSProtocol()) 730 | 731 | // Do not open the firewall implicitly, we always create explicit firewall rules 732 | p.SetOpenfirewall(false) 733 | 734 | // Read the source CIDR annotation 735 | cidrList, err := lb.getCIDRList(service) 736 | if err != nil { 737 | return nil, err 738 | } 739 | 740 | // Set the CIDR list in the parameters 741 | p.SetCidrlist(cidrList) 742 | 743 | // Create a new load balancer rule. 744 | r, err := lb.LoadBalancer.CreateLoadBalancerRule(p) 745 | if err != nil { 746 | return nil, fmt.Errorf("error creating load balancer rule %v: %v", lbRuleName, err) 747 | } 748 | 749 | lbRule := &cloudstack.LoadBalancerRule{ 750 | Id: r.Id, 751 | Algorithm: r.Algorithm, 752 | Cidrlist: r.Cidrlist, 753 | Name: r.Name, 754 | Networkid: r.Networkid, 755 | Privateport: r.Privateport, 756 | Publicport: r.Publicport, 757 | Publicip: r.Publicip, 758 | Publicipid: r.Publicipid, 759 | Protocol: r.Protocol, 760 | } 761 | 762 | return lbRule, nil 763 | } 764 | 765 | // deleteLoadBalancerRule deletes a load balancer rule. 766 | func (lb *loadBalancer) deleteLoadBalancerRule(lbRule *cloudstack.LoadBalancerRule) error { 767 | p := lb.LoadBalancer.NewDeleteLoadBalancerRuleParams(lbRule.Id) 768 | 769 | if _, err := lb.LoadBalancer.DeleteLoadBalancerRule(p); err != nil { 770 | return fmt.Errorf("error deleting load balancer rule %v: %v", lbRule.Name, err) 771 | } 772 | 773 | // Delete the rule from the map as it no longer exists 774 | delete(lb.rules, lbRule.Name) 775 | 776 | return nil 777 | } 778 | 779 | // assignHostsToRule assigns hosts to a load balancer rule. 780 | func (lb *loadBalancer) assignHostsToRule(lbRule *cloudstack.LoadBalancerRule, hostIDs []string) error { 781 | p := lb.LoadBalancer.NewAssignToLoadBalancerRuleParams(lbRule.Id) 782 | p.SetVirtualmachineids(hostIDs) 783 | 784 | if _, err := lb.LoadBalancer.AssignToLoadBalancerRule(p); err != nil { 785 | return fmt.Errorf("error assigning hosts to load balancer rule %v: %v", lbRule.Name, err) 786 | } 787 | 788 | return nil 789 | } 790 | 791 | // removeHostsFromRule removes hosts from a load balancer rule. 792 | func (lb *loadBalancer) removeHostsFromRule(lbRule *cloudstack.LoadBalancerRule, hostIDs []string) error { 793 | p := lb.LoadBalancer.NewRemoveFromLoadBalancerRuleParams(lbRule.Id) 794 | p.SetVirtualmachineids(hostIDs) 795 | 796 | if _, err := lb.LoadBalancer.RemoveFromLoadBalancerRule(p); err != nil { 797 | return fmt.Errorf("error removing hosts from load balancer rule %v: %v", lbRule.Name, err) 798 | } 799 | 800 | return nil 801 | } 802 | 803 | // symmetricDifference returns the symmetric difference between the old (existing) and new (wanted) host ID's. 804 | func symmetricDifference(hostIDs []string, lbInstances []*cloudstack.VirtualMachine) ([]string, []string) { 805 | new := make(map[string]bool) 806 | for _, hostID := range hostIDs { 807 | new[hostID] = true 808 | } 809 | 810 | var remove []string 811 | for _, instance := range lbInstances { 812 | if new[instance.Id] { 813 | delete(new, instance.Id) 814 | continue 815 | } 816 | 817 | remove = append(remove, instance.Id) 818 | } 819 | 820 | var assign []string 821 | for hostID := range new { 822 | assign = append(assign, hostID) 823 | } 824 | 825 | return assign, remove 826 | } 827 | 828 | // compareStringSlice compares two unsorted slices of strings without sorting them first. 829 | // 830 | // The slices are equal if and only if both contain the same number of every unique element. 831 | // 832 | // Thanks to: https://stackoverflow.com/a/36000696 833 | func compareStringSlice(x, y []string) bool { 834 | if len(x) != len(y) { 835 | return false 836 | } 837 | // create a map of string -> int 838 | diff := make(map[string]int, len(x)) 839 | for _, _x := range x { 840 | // 0 value for int is 0, so just increment a counter for the string 841 | diff[_x]++ 842 | } 843 | for _, _y := range y { 844 | // If the string _y is not in diff bail out early 845 | if _, ok := diff[_y]; !ok { 846 | return false 847 | } 848 | diff[_y] -= 1 849 | if diff[_y] == 0 { 850 | delete(diff, _y) 851 | } 852 | } 853 | return len(diff) == 0 854 | } 855 | 856 | func ruleToString(rule *cloudstack.FirewallRule) string { 857 | ls := &strings.Builder{} 858 | if rule == nil { 859 | ls.WriteString("nil") 860 | } else { 861 | switch rule.Protocol { 862 | case "tcp": 863 | fallthrough 864 | case "udp": 865 | fmt.Fprintf(ls, "{[%s] -> %s:[%d-%d] (%s)}", rule.Cidrlist, rule.Ipaddress, rule.Startport, rule.Endport, rule.Protocol) 866 | case "icmp": 867 | fmt.Fprintf(ls, "{[%s] -> %s [%d,%d] (%s)}", rule.Cidrlist, rule.Ipaddress, rule.Icmptype, rule.Icmpcode, rule.Protocol) 868 | default: 869 | fmt.Fprintf(ls, "{[%s] -> %s (%s)}", rule.Cidrlist, rule.Ipaddress, rule.Protocol) 870 | } 871 | } 872 | return ls.String() 873 | } 874 | 875 | func rulesToString(rules []*cloudstack.FirewallRule) string { 876 | ls := &strings.Builder{} 877 | first := true 878 | for _, rule := range rules { 879 | if first { 880 | first = false 881 | } else { 882 | ls.WriteString(", ") 883 | } 884 | ls.WriteString(ruleToString(rule)) 885 | } 886 | return ls.String() 887 | } 888 | 889 | func rulesMapToString(rules map[*cloudstack.FirewallRule]bool) string { 890 | ls := &strings.Builder{} 891 | first := true 892 | for rule := range rules { 893 | if first { 894 | first = false 895 | } else { 896 | ls.WriteString(", ") 897 | } 898 | ls.WriteString(ruleToString(rule)) 899 | } 900 | return ls.String() 901 | } 902 | 903 | // updateFirewallRule creates a firewall rule for a load balancer rule 904 | // 905 | // If the rule list is empty, all internet (IPv4: 0.0.0.0/0) is opened for the 906 | // load balancer's port+protocol implicitly. 907 | // 908 | // Returns true if the firewall rule was created or updated 909 | func (lb *loadBalancer) updateFirewallRule(publicIpId string, publicPort int, protocol LoadBalancerProtocol, allowedIPs []string) (bool, error) { 910 | if len(allowedIPs) == 0 { 911 | allowedIPs = []string{defaultAllowedCIDR} 912 | } 913 | 914 | p := lb.Firewall.NewListFirewallRulesParams() 915 | p.SetIpaddressid(publicIpId) 916 | p.SetListall(true) 917 | if lb.projectID != "" { 918 | p.SetProjectid(lb.projectID) 919 | } 920 | klog.V(4).Infof("Listing firewall rules for %v", p) 921 | r, err := lb.Firewall.ListFirewallRules(p) 922 | if err != nil { 923 | return false, fmt.Errorf("error fetching firewall rules for public IP %v: %v", publicIpId, err) 924 | } 925 | klog.V(4).Infof("All firewall rules for %v: %v", lb.ipAddr, rulesToString(r.FirewallRules)) 926 | 927 | // find all rules that have a matching proto+port 928 | // a map may or may not be faster, but is a bit easier to understand 929 | filtered := make(map[*cloudstack.FirewallRule]bool) 930 | for _, rule := range r.FirewallRules { 931 | if rule.Protocol == protocol.IPProtocol() && rule.Startport == publicPort && rule.Endport == publicPort { 932 | filtered[rule] = true 933 | } 934 | } 935 | klog.V(4).Infof("Matching rules for %v: %v", lb.ipAddr, rulesMapToString(filtered)) 936 | 937 | // determine if we already have a rule with matching cidrs 938 | var match *cloudstack.FirewallRule 939 | for rule := range filtered { 940 | cidrlist := strings.Split(rule.Cidrlist, ",") 941 | if compareStringSlice(cidrlist, allowedIPs) { 942 | klog.V(4).Infof("Found identical rule: %v", rule) 943 | match = rule 944 | break 945 | } 946 | } 947 | 948 | if match != nil { 949 | // no need to create a new rule - but prevent deletion of the matching rule 950 | delete(filtered, match) 951 | } 952 | 953 | // delete all other rules that didn't match the CIDR list 954 | // do this first to prevent CS rule conflict errors 955 | klog.V(4).Infof("Firewall rules to be deleted for %v: %v", lb.ipAddr, rulesMapToString(filtered)) 956 | for rule := range filtered { 957 | p := lb.Firewall.NewDeleteFirewallRuleParams(rule.Id) 958 | _, err = lb.Firewall.DeleteFirewallRule(p) 959 | if err != nil { 960 | // report the error, but keep on deleting the other rules 961 | klog.Errorf("Error deleting old firewall rule %v: %v", rule.Id, err) 962 | } 963 | } 964 | 965 | // create new rule if necessary 966 | if match == nil { 967 | // no rule found, create a new one 968 | p := lb.Firewall.NewCreateFirewallRuleParams(publicIpId, protocol.IPProtocol()) 969 | p.SetCidrlist(allowedIPs) 970 | p.SetStartport(publicPort) 971 | p.SetEndport(publicPort) 972 | _, err = lb.Firewall.CreateFirewallRule(p) 973 | if err != nil { 974 | // return immediately if we can't create the new rule 975 | return false, fmt.Errorf("error creating new firewall rule for public IP %v, proto %v, port %v, allowed %v: %v", publicIpId, protocol, publicPort, allowedIPs, err) 976 | } 977 | } 978 | 979 | // return true (because we changed something), but also the last error if deleting one old rule failed 980 | return true, err 981 | } 982 | 983 | func (lb *loadBalancer) updateNetworkACL(publicPort int, protocol LoadBalancerProtocol, networkId string) (bool, error) { 984 | network, _, err := lb.Network.GetNetworkByID(networkId) 985 | if err != nil { 986 | return false, fmt.Errorf("error fetching Network with ID: %v, due to: %s", networkId, err) 987 | } 988 | 989 | networkAclList, count, err := lb.NetworkACL.GetNetworkACLListByID(network.Aclid) 990 | if err != nil { 991 | return false, fmt.Errorf("error fetching Network ACL List with ID: %v, due to: %s", network.Aclid, err) 992 | } 993 | 994 | if count == 0 { 995 | return false, fmt.Errorf("failed to find network ACL List with id: %v", network.Aclid) 996 | } 997 | 998 | if networkAclList.Name == "default_allow" || networkAclList.Name == "default_deny" { 999 | klog.Infof("Network is using a default network ACL. Cannot add ACL rules to default ACLs") 1000 | return true, err 1001 | } 1002 | 1003 | networkAclParams := lb.NetworkACL.NewListNetworkACLsParams() 1004 | networkAclParams.SetAclid(network.Aclid) 1005 | networkAclParams.SetNetworkid(networkId) 1006 | 1007 | networkAclResponse, err := lb.NetworkACL.ListNetworkACLs(networkAclParams) 1008 | 1009 | if err != nil { 1010 | return false, fmt.Errorf("error fetching Network ACL with ID: %v for network with id: %v, due to: %s", network.Aclid, networkId, err) 1011 | } 1012 | 1013 | // find all network ACL rules that have a matching proto+port 1014 | // a map may or may not be faster, but is a bit easier to understand 1015 | filtered := make(map[*cloudstack.NetworkACL]bool) 1016 | for _, netAclRule := range networkAclResponse.NetworkACLs { 1017 | if netAclRule.Protocol == protocol.IPProtocol() && netAclRule.Startport == strconv.Itoa(publicPort) && netAclRule.Endport == strconv.Itoa(publicPort) { 1018 | filtered[netAclRule] = true 1019 | } 1020 | } 1021 | 1022 | if len(filtered) > 0 { 1023 | klog.V(4).Infof("Network ACL rule for port %v and protocol %v already exists. No need to added a duplicate rule", publicPort, protocol) 1024 | return true, err 1025 | } 1026 | 1027 | // create ACL rule 1028 | acl := lb.NetworkACL.NewCreateNetworkACLParams(protocol.CSProtocol()) 1029 | acl.SetAclid(network.Aclid) 1030 | acl.SetAction("Allow") 1031 | acl.SetCidrlist([]string{"0.0.0.0/0"}) 1032 | acl.SetStartport(publicPort) 1033 | acl.SetEndport(publicPort) 1034 | acl.SetNetworkid(networkId) 1035 | acl.SetTraffictype("Ingress") 1036 | 1037 | _, err = lb.NetworkACL.CreateNetworkACL(acl) 1038 | if err != nil { 1039 | return false, fmt.Errorf("error creating Network ACL for port: %v, due to: %s", publicPort, err) 1040 | } 1041 | return true, err 1042 | } 1043 | 1044 | // deleteFirewallRule deletes the firewall rule associated with the ip:port:protocol combo 1045 | // 1046 | // returns true when corresponding rules were deleted 1047 | func (lb *loadBalancer) deleteFirewallRule(publicIpId string, publicPort int, protocol LoadBalancerProtocol) (bool, error) { 1048 | p := lb.Firewall.NewListFirewallRulesParams() 1049 | p.SetIpaddressid(publicIpId) 1050 | p.SetListall(true) 1051 | if lb.projectID != "" { 1052 | p.SetProjectid(lb.projectID) 1053 | } 1054 | r, err := lb.Firewall.ListFirewallRules(p) 1055 | if err != nil { 1056 | return false, fmt.Errorf("error fetching firewall rules for public IP %v: %v", publicIpId, err) 1057 | } 1058 | 1059 | // filter by proto:port 1060 | filtered := make([]*cloudstack.FirewallRule, 0, 1) 1061 | for _, rule := range r.FirewallRules { 1062 | if rule.Protocol == protocol.IPProtocol() && rule.Startport == publicPort && rule.Endport == publicPort { 1063 | filtered = append(filtered, rule) 1064 | } 1065 | } 1066 | 1067 | // delete all rules 1068 | deleted := false 1069 | for _, rule := range filtered { 1070 | p := lb.Firewall.NewDeleteFirewallRuleParams(rule.Id) 1071 | _, err = lb.Firewall.DeleteFirewallRule(p) 1072 | if err != nil { 1073 | klog.Errorf("Error deleting old firewall rule %v: %v", rule.Id, err) 1074 | } else { 1075 | deleted = true 1076 | } 1077 | } 1078 | 1079 | return deleted, err 1080 | } 1081 | 1082 | // Delete Network ACLs deletes the Network ACL rule associated with the ip:port:protocol combo 1083 | func (lb *loadBalancer) deleteNetworkACLRule(publicPort int, protocol LoadBalancerProtocol, networkID string) (bool, error) { 1084 | p := lb.NetworkACL.NewListNetworkACLsParams() 1085 | p.SetListall(true) 1086 | p.SetNetworkid(networkID) 1087 | if lb.projectID != "" { 1088 | p.SetProjectid(lb.projectID) 1089 | } 1090 | 1091 | r, err := lb.NetworkACL.ListNetworkACLs(p) 1092 | if err != nil { 1093 | return false, fmt.Errorf("error fetching Network ACL rules Network ID %v: %v", networkID, err) 1094 | } 1095 | 1096 | // filter by proto:port 1097 | filtered := make([]*cloudstack.NetworkACL, 0, 1) 1098 | for _, rule := range r.NetworkACLs { 1099 | if rule.Protocol == protocol.IPProtocol() && rule.Startport == strconv.Itoa(publicPort) && rule.Endport == strconv.Itoa(publicPort) { 1100 | filtered = append(filtered, rule) 1101 | } 1102 | } 1103 | 1104 | // delete first filtered rules 1105 | if len(filtered) == 0 { 1106 | klog.V(4).Infof("No ACL rules found matching protocol: %v and port: %v", protocol, publicPort) 1107 | return true, nil 1108 | } 1109 | deleted := false 1110 | ruleToBeDeleted := filtered[0] 1111 | deleteAclParams := lb.NetworkACL.NewDeleteNetworkACLParams(ruleToBeDeleted.Id) 1112 | _, err = lb.NetworkACL.DeleteNetworkACL(deleteAclParams) 1113 | if err != nil { 1114 | klog.Errorf("Error deleting old Network ACL rule %v: %v", ruleToBeDeleted.Id, err) 1115 | } else { 1116 | deleted = true 1117 | } 1118 | 1119 | return deleted, err 1120 | } 1121 | 1122 | // getStringFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's value or a specified defaultSetting 1123 | func getStringFromServiceAnnotation(service *corev1.Service, annotationKey string, defaultSetting string) string { 1124 | klog.V(4).Infof("getStringFromServiceAnnotation(%s/%s, %v, %v)", service.Namespace, service.Name, annotationKey, defaultSetting) 1125 | if annotationValue, ok := service.Annotations[annotationKey]; ok { 1126 | //if there is an annotation for this setting, set the "setting" var to it 1127 | // annotationValue can be empty, it is working as designed 1128 | // it makes possible for instance provisioning loadbalancer without floatingip 1129 | klog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, annotationValue) 1130 | return annotationValue 1131 | } 1132 | //if there is no annotation, set "settings" var to the value from cloud config 1133 | if defaultSetting != "" { 1134 | klog.V(4).Infof("Could not find a Service Annotation; falling back on cloud-config setting: %v = %v", annotationKey, defaultSetting) 1135 | } 1136 | return defaultSetting 1137 | } 1138 | 1139 | // getBoolFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's boolean value or a specified defaultSetting 1140 | func getBoolFromServiceAnnotation(service *corev1.Service, annotationKey string, defaultSetting bool) bool { 1141 | klog.V(4).Infof("getBoolFromServiceAnnotation(%s/%s, %v, %v)", service.Namespace, service.Name, annotationKey, defaultSetting) 1142 | if annotationValue, ok := service.Annotations[annotationKey]; ok { 1143 | returnValue := false 1144 | switch annotationValue { 1145 | case "true": 1146 | returnValue = true 1147 | case "false": 1148 | returnValue = false 1149 | default: 1150 | returnValue = defaultSetting 1151 | } 1152 | 1153 | klog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, returnValue) 1154 | return returnValue 1155 | } 1156 | klog.V(4).Infof("Could not find a Service Annotation; falling back to default setting: %v = %v", annotationKey, defaultSetting) 1157 | return defaultSetting 1158 | } 1159 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= 9 | cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= 10 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= 11 | cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= 12 | cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= 13 | cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= 14 | cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= 15 | cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= 16 | cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= 17 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 18 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= 19 | cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= 20 | cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= 21 | cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= 22 | cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= 23 | cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= 24 | cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= 25 | cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= 26 | cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= 27 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 28 | cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= 29 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 30 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= 31 | cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= 32 | cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= 33 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 34 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= 35 | cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= 36 | cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= 37 | cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= 38 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 39 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= 40 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 41 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 42 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 43 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 44 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 45 | github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= 46 | github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= 47 | github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 48 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 49 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 50 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 51 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 52 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 53 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 54 | github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= 55 | github.com/apache/cloudstack-go/v2 v2.19.0 h1:YHLw770MmgiqXx6NRFYw2Nr7DpnylLhLG2KYNCftgnc= 56 | github.com/apache/cloudstack-go/v2 v2.19.0/go.mod h1:p/YBUwIEkQN6CQxFhw8Ff0wzf1MY0qRRRuGYNbcb1F8= 57 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 58 | github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= 59 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 60 | github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 61 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 62 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 63 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 64 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 65 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= 66 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= 67 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 68 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 69 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 70 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 71 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 72 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 73 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 74 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 75 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 76 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 77 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= 78 | github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= 79 | github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= 80 | github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= 81 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 82 | github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= 83 | github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 84 | github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 85 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 86 | github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= 87 | github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 88 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 89 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 90 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 91 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= 92 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 93 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 94 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 95 | github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= 96 | github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 97 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 98 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 99 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 100 | github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= 101 | github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= 102 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 103 | github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= 104 | github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= 105 | github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= 106 | github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 107 | github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 108 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 109 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 110 | github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= 111 | github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= 112 | github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= 113 | github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= 114 | github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= 115 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 116 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 117 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 118 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 119 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 120 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 121 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 122 | github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= 123 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 124 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 125 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 126 | github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 127 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 128 | github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= 129 | github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 130 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 131 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 132 | github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 133 | github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= 134 | github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 135 | github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= 136 | github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= 137 | github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= 138 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 139 | github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= 140 | github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= 141 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 142 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 143 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 144 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 145 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 146 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 147 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 148 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 149 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 150 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 151 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 152 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 153 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 154 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 155 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 156 | github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 157 | github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 158 | github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= 159 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 160 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 161 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 162 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 163 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 164 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 165 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 166 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 167 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 168 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 169 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 170 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 171 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 172 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 173 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 174 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 175 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 176 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 177 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 178 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 179 | github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= 180 | github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= 181 | github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= 182 | github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= 183 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 184 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 185 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 186 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 187 | github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 188 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 189 | github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 190 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 191 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 192 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 193 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 194 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 195 | github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= 196 | github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 197 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 198 | github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= 199 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 200 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 201 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 202 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 203 | github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 204 | github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 205 | github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 206 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 207 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 208 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 209 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 210 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 211 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 212 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 213 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 214 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 215 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= 216 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= 217 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= 218 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 219 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= 220 | github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= 221 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 222 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 223 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 224 | github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= 225 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 226 | github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= 227 | github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 228 | github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= 229 | github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= 230 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 231 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 232 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 233 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 234 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 235 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 236 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 237 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 238 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 239 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 240 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 241 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 242 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 243 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 244 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 245 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 246 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 247 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 248 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 249 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 250 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 251 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 252 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 253 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 254 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 255 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 256 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 257 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 258 | github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= 259 | github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 260 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 261 | github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= 262 | github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 263 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 264 | github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= 265 | github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= 266 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 267 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 268 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 269 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 270 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 271 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 272 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 273 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 274 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 275 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 276 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 277 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 278 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 279 | github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= 280 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 281 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 282 | github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= 283 | github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 284 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 285 | github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= 286 | github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= 287 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 288 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 289 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 290 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 291 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 292 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 293 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 294 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 295 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 296 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 297 | github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= 298 | github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= 299 | github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= 300 | github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= 301 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 302 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 303 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 304 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 305 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 306 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 307 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 308 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 309 | github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= 310 | github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= 311 | github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= 312 | github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= 313 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 314 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 315 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 316 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 317 | github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 318 | github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= 319 | github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= 320 | github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= 321 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 322 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 323 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 324 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 325 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 326 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 327 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 328 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= 329 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 330 | github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= 331 | github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= 332 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 333 | github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= 334 | github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= 335 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 336 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 337 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 338 | github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= 339 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 340 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 341 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 342 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 343 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 344 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 345 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 346 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 347 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 348 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 349 | github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= 350 | github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 351 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= 352 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 353 | go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= 354 | go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= 355 | go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0= 356 | go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU= 357 | go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ= 358 | go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI= 359 | go.etcd.io/etcd/client/v2 v2.305.0 h1:ftQ0nOOHMcbMS3KIaDQ0g5Qcd6bhaBrQT6b89DfwLTs= 360 | go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= 361 | go.etcd.io/etcd/client/v3 v3.5.14 h1:CWfRs4FDaDoSz81giL7zPpZH2Z35tbOrAJkkjMqOupg= 362 | go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk= 363 | go.etcd.io/etcd/pkg/v3 v3.5.0 h1:ntrg6vvKRW26JRmHTE0iNlDgYK6JX3hg/4cD62X0ixk= 364 | go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= 365 | go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= 366 | go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= 367 | go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= 368 | go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= 369 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 370 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 371 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 372 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 373 | go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 374 | go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= 375 | go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= 376 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= 377 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= 378 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= 379 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= 380 | go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= 381 | go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= 382 | go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= 383 | go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= 384 | go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= 385 | go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= 386 | go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= 387 | go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= 388 | go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= 389 | go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= 390 | go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= 391 | go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= 392 | go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= 393 | go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= 394 | go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= 395 | go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= 396 | go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= 397 | go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= 398 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 399 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 400 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= 401 | go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= 402 | go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= 403 | go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= 404 | go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= 405 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 406 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 407 | go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= 408 | go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= 409 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 410 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 411 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 412 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 413 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 414 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 415 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 416 | golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= 417 | golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= 418 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 419 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 420 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 421 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 422 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 423 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 424 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 425 | golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 426 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= 427 | golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= 428 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 429 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 430 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 431 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 432 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 433 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 434 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 435 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 436 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 437 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 438 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 439 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 440 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 441 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 442 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 443 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 444 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 445 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 446 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 447 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 448 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 449 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 450 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 451 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 452 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 453 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 454 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 455 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 456 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 457 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 458 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 459 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 460 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 461 | golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 462 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 463 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 464 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 465 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 466 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 467 | golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 468 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 469 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 470 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 471 | golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 472 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 473 | golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 474 | golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 475 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 476 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 477 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 478 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 479 | golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 480 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 481 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 482 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 483 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 484 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 485 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 486 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 487 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 488 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 489 | golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= 490 | golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= 491 | golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= 492 | golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= 493 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= 494 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 495 | golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw= 496 | golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 497 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 498 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 499 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 500 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 501 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 502 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 503 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 504 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 505 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 506 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 507 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 508 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 509 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 510 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 511 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 512 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 513 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 514 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 515 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 516 | golang.org/x/tools v0.0.0-20190313210603-aa82965741a9/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 517 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 518 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 519 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 520 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 521 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 522 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 523 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 524 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 525 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 526 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 527 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 528 | google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 529 | google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 530 | google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 531 | google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 532 | google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 533 | google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 534 | google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= 535 | google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= 536 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 537 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 538 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 539 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 540 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 541 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 542 | google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= 543 | google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= 544 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 545 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 546 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 547 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 548 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 549 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 550 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 551 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 552 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 553 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 554 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 555 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 556 | google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 557 | google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 558 | google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= 559 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 560 | google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 561 | google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 562 | google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 563 | google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 564 | google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 565 | google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 566 | google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 567 | google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 568 | google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= 569 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 570 | google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= 571 | google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 572 | google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 573 | google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 574 | google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 575 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= 576 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= 577 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= 578 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= 579 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 h1:Q2RxlXqh1cgzzUgV261vBO2jI5R/3DD1J2pM0nI4NhU= 580 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= 581 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 582 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 583 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 584 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 585 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 586 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 587 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 588 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 589 | google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= 590 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 591 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 592 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 593 | google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= 594 | google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= 595 | google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= 596 | google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= 597 | google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= 598 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 599 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 600 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 601 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 602 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 603 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 604 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 605 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 606 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 607 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 608 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 609 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 610 | google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= 611 | google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 612 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 613 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 614 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 615 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 616 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 617 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 618 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 619 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 620 | gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= 621 | gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= 622 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 623 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 624 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= 625 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 626 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 627 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 628 | gopkg.in/warnings.v0 v0.1.1 h1:XM28wIgFzaBmeZ5dNHIpWLQpt/9DGKxk+rCg/22nnYE= 629 | gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 630 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 631 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 632 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 633 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 634 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 635 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 636 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 637 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 638 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 639 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 640 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 641 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 642 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 643 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 644 | gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= 645 | gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= 646 | gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= 647 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 648 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 649 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 650 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 651 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 652 | honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 653 | honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 654 | k8s.io/api v0.24.17 h1:ILPpMleNDZbMJwopUBOVWtmCq3xBAj/4gJEUicy6QGs= 655 | k8s.io/api v0.24.17/go.mod h1:Ff5rnpz9qMj3/tXXA504wdk7Mf9zW3JSNWp5tf80VMQ= 656 | k8s.io/apimachinery v0.24.17 h1:mewWCeZ3Swr4EAfatVAhHXJHGzCHojphWA/5UJW4pPY= 657 | k8s.io/apimachinery v0.24.17/go.mod h1:kSzhCwldu9XB172NDdLffRN0sJ3x95RR7Bmyc4SHhs0= 658 | k8s.io/apiserver v0.24.17 h1:APbTxPHcOXkorGZXUtA8esA938W9choQA9z9NUDmqm4= 659 | k8s.io/apiserver v0.24.17/go.mod h1:qr6HYxuodmqma+f4f2ck1lahy3bgJQ+8A17YPJKiJNg= 660 | k8s.io/client-go v0.24.17 h1:NqBXp0NNa6wYpg6VEeaeBc202OUdum6cd+R/OelhQCU= 661 | k8s.io/client-go v0.24.17/go.mod h1:MPiIOfyXDQZXKHKZZh+MuY1huqJLNUAqARaJO6i4nwY= 662 | k8s.io/cloud-provider v0.24.17 h1:GNCOb2NsbNDilDNJZOFFMlbIgd9UR4VjlNHPxkUYIT8= 663 | k8s.io/cloud-provider v0.24.17/go.mod h1:/c8FpCIT8mJ7fszFDbV2Go9yC+Gquxu9ZX6EdIzSF6Q= 664 | k8s.io/component-base v0.24.17 h1:BY00bMItlHOeRydQDpE0txIbBqEjwBZ0mvPbY3IqJJc= 665 | k8s.io/component-base v0.24.17/go.mod h1:75f8qsjuz9NxxI66S9NmHJwR2kMYUj4JtvrzC+7l3lo= 666 | k8s.io/component-helpers v0.24.17 h1:rqYMHM170FulYKtienTmnkFzLIOkzUe7k8Ro/KTH83Y= 667 | k8s.io/component-helpers v0.24.17/go.mod h1:J1jKEUypUVOIR7X6OvMK4nEzUgcD0Mx+jHt0I1TYz9Q= 668 | k8s.io/controller-manager v0.24.17 h1:vP3V2p0DHdLmgiF9bSU/91+bU+xHInzfWBf45zuXKRY= 669 | k8s.io/controller-manager v0.24.17/go.mod h1:omLQjgRvJGNYbVk7YEuND4tmb4GGOTpO1tLEyzyopxU= 670 | k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= 671 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 672 | k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= 673 | k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= 674 | k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= 675 | k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= 676 | k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= 677 | k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 678 | k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= 679 | k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 680 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 681 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= 682 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 683 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 h1:fAPTNEpzQMOLMGwOHNbUkR2xXTQwMJOZYNx+/mLlOh0= 684 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37/go.mod h1:vfnxT4FXNT8eGvO+xi/DsyC/qHmdujqwrUa1WSspCsk= 685 | sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= 686 | sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 687 | sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= 688 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= 689 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= 690 | sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= 691 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 692 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 693 | --------------------------------------------------------------------------------