├── internal ├── ipvs │ ├── doc.go │ ├── ns │ │ ├── doc.go │ │ └── init_linux.go │ ├── .gitignore │ ├── README.md │ ├── netlink_linux_test.go │ ├── constants_linux.go │ ├── ipvs_linux.go │ ├── LICENSE │ ├── ipvs_linux_test.go │ └── netlink_linux.go ├── ipam │ ├── types.go │ └── simple.go ├── loadbalancer │ ├── impl_test.go │ ├── loadbalancer.proto │ ├── loadbalancer_grpc.pb.go │ ├── impl.go │ └── loadbalancer.pb.go └── proxmox │ ├── provider.go │ ├── instance.go │ └── loadbalancer.go ├── chart ├── templates │ ├── serviceaccount.yaml │ ├── clusterroles.yaml │ ├── lbctl-secret.yaml │ ├── cloud-config.yaml │ ├── _helpers.tpl │ └── daemonset.yaml ├── .helmignore ├── Chart.yaml └── values.yaml ├── Dockerfile ├── lbmanager.yaml ├── cmd ├── lbmanager │ └── main.go └── lbctl │ └── main.go ├── main.go ├── scripts └── installLB.sh ├── go.mod ├── README.md └── LICENSE.txt /internal/ipvs/doc.go: -------------------------------------------------------------------------------- 1 | package ipvs 2 | -------------------------------------------------------------------------------- /internal/ipvs/ns/doc.go: -------------------------------------------------------------------------------- 1 | package ns 2 | -------------------------------------------------------------------------------- /internal/ipam/types.go: -------------------------------------------------------------------------------- 1 | package ipam 2 | 3 | import "net" 4 | 5 | type IPAM interface { 6 | Allocate() (net.IP, error) 7 | Release(ip net.IP) error 8 | IsUsed(ip net.IP) bool 9 | Contains(ip net.IP) bool 10 | } 11 | -------------------------------------------------------------------------------- /chart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "chart.serviceAccountName" . }} 6 | labels: 7 | {{- include "chart.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 as builder 2 | 3 | WORKDIR /app 4 | COPY go.mod go.sum ./ 5 | RUN go mod download 6 | 7 | COPY main.go . 8 | COPY cmd ./cmd 9 | COPY internal ./internal 10 | RUN go build -o proxmox-cloud-controller-manager . 11 | 12 | #FROM gcr.io/distroless/base-debian12:latest 13 | FROM debian:stable 14 | 15 | COPY --from=builder /app/proxmox-cloud-controller-manager / 16 | 17 | CMD ["/proxmox-cloud-controller-manager"] 18 | -------------------------------------------------------------------------------- /chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /chart/templates/clusterroles.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: {{ include "chart.name" . }} 6 | labels: 7 | {{- include "chart.labels" . | nindent 4 }} 8 | 9 | roleRef: 10 | apiGroup: rbac.authorization.k8s.io 11 | kind: ClusterRole 12 | name: cluster-admin 13 | subjects: 14 | - kind: ServiceAccount 15 | name: {{ include "chart.serviceAccountName" . }} 16 | namespace: {{ .Release.Namespace }} 17 | 18 | -------------------------------------------------------------------------------- /internal/ipvs/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *~ 6 | .gtm 7 | tags 8 | .DS_Store 9 | 10 | # Folders 11 | _obj 12 | _test 13 | 14 | 15 | # Architecture specific extensions/prefixes 16 | *.[568vq] 17 | [568vq].out 18 | 19 | *.cgo1.go 20 | *.cgo2.c 21 | _cgo_defun.c 22 | _cgo_gotypes.go 23 | _cgo_export.* 24 | 25 | _testmain.go 26 | 27 | *.exe 28 | *.test 29 | *.prof 30 | 31 | # Coverage 32 | *.tmp 33 | *.coverprofile 34 | 35 | # IDE files and folders 36 | .project 37 | .settings/ 38 | .idea/ 39 | -------------------------------------------------------------------------------- /chart/templates/lbctl-secret.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "chart.fullname" . }}-promox-lbctl 6 | labels: 7 | {{- include "chart.labels" . | nindent 4 }} 8 | data: 9 | tls.key: {{ .Values.lbmanager.tls.key | b64enc | quote }} 10 | tls.crt: {{ .Values.lbmanager.tls.cert | b64enc | quote }} 11 | ca.crt: {{ .Values.lbmanager.tls.caCert | b64enc | quote }} 12 | proxmoxToken: {{ .Values.proxmox.apiToken | b64enc | quote }} 13 | proxmoxUser: {{ .Values.proxmox.user | b64enc | quote }} 14 | proxmoxCA.crt: {{ .Values.proxmox.caCert | b64enc | quote }} 15 | 16 | -------------------------------------------------------------------------------- /lbmanager.yaml: -------------------------------------------------------------------------------- 1 | grpc: 2 | listen: 0.0.0.0 3 | port: 9999 4 | auth: 5 | key: certs/lbmanager-dev.pem 6 | cert: certs/lbmanager-dev.pem 7 | ca: certs/ca.pem 8 | dbDir: /tmp/db 9 | loadbalancer: 10 | namespace: LB 11 | externalInterface: dummy0 12 | internalInterface: eth1 13 | ipam: 14 | # Simple IPAM 15 | 16 | # cidr is the network in which this IPAM operates 17 | cidr: 192.168.87.0/24 18 | # dynamicRange is the range from which random LB addresses 19 | # will be taken. 20 | # It's valid to explicitly ask for an IP address that is 21 | # outside this range, as long as it's inside the cidr. 22 | # The entire dynamicRange must be contained in the cidr. 23 | dynamicRange: 24 | startAt: 192.168.87.200 25 | endAt: 192.168.87.225 26 | 27 | -------------------------------------------------------------------------------- /internal/loadbalancer/impl_test.go: -------------------------------------------------------------------------------- 1 | package loadbalancer 2 | 3 | import ( 4 | context "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/knadh/koanf/parsers/yaml" 9 | "github.com/knadh/koanf/providers/file" 10 | "github.com/knadh/koanf/v2" 11 | ) 12 | 13 | var k = koanf.New(".") 14 | 15 | func TestCreateLB(t *testing.T) { 16 | config := file.Provider("../../lbmanager.yaml") 17 | if err := k.Load(config, yaml.Parser()); err != nil { 18 | t.Error(err) 19 | } 20 | 21 | lbServer, err := NewServer(k) 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | defer lbServer.Close() 26 | 27 | lbi, err := lbServer.Create(context.TODO(), &CreateLoadBalancer{ 28 | Name: "test1", 29 | IpAddr: asPtr("10.0.0.1"), 30 | }) 31 | if err != nil { 32 | t.Error(err) 33 | } 34 | fmt.Printf("%+v\n", lbi) 35 | } 36 | -------------------------------------------------------------------------------- /chart/templates/cloud-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "chart.fullname" . }}-cloud-config 6 | labels: 7 | {{- include "chart.labels" . | nindent 4 }} 8 | data: 9 | "cloud-config.yaml": | 10 | security: 11 | cert: /var/run/secrets/ccm/tls.crt 12 | key: /var/run/secrets/ccm/tls.key 13 | ca: /var/run/secrets/ccm/ca.crt 14 | managerHost: {{ .Values.lbmanager.hostname }} 15 | managerPort: {{ default "9999" .Values.lbmanager.port }} 16 | proxmoxConfig: 17 | url: {{ .Values.proxmox.apiURL }} 18 | timeout: {{ default "5" .Values.proxmox.timeout }} 19 | apiToken: /var/run/secrets/ccm/proxmoxToken 20 | username: /var/run/secrets/ccm/proxmoxUser 21 | insecureTLS: {{ default "false" .Values.proxmox.insecure }} 22 | caCert: /var/run/secrets/ccm/proxmoxCA.crt 23 | -------------------------------------------------------------------------------- /chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: proxmox-cloud-provider 3 | description: A Helm chart for installing the Proxmox Cloud Provider 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.0.0" 25 | -------------------------------------------------------------------------------- /internal/ipvs/README.md: -------------------------------------------------------------------------------- 1 | 2 | This is a copy of github.com/moby/ipvs because it seems that the moby project is not maintained anymore. 3 | 4 | All rights and copyright on this folder are as specified below. 5 | 6 | # ipvs - networking for containers 7 | 8 | ![Test](https://github.com/moby/ipvs/workflows/Test/badge.svg) [![GoDoc](https://godoc.org/github.com/moby/ipvs?status.svg)](https://godoc.org/github.com/moby/ipvs) [![Go Report Card](https://goreportcard.com/badge/github.com/moby/ipvs)](https://goreportcard.com/report/github.com/moby/ipvs) 9 | 10 | ipvs provides a native Go implementation for communicating with IPVS kernel module using a netlink socket. 11 | 12 | 13 | 14 | #### Using ipvs 15 | 16 | ```go 17 | import ( 18 | "log" 19 | 20 | "github.com/moby/ipvs" 21 | ) 22 | 23 | func main() { 24 | handle, err := ipvs.New("") 25 | if err != nil { 26 | log.Fatalf("ipvs.New: %s", err) 27 | } 28 | svcs, err := handle.GetServices() 29 | if err != nil { 30 | log.Fatalf("handle.GetServices: %s", err) 31 | } 32 | } 33 | ``` 34 | 35 | ## Contributing 36 | 37 | Want to hack on ipvs? [Docker's contributions guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md) apply. 38 | 39 | ## Copyright and license 40 | 41 | Copyright 2015 Docker, inc. Code released under the [Apache 2.0 license](LICENSE). 42 | -------------------------------------------------------------------------------- /internal/ipvs/netlink_linux_test.go: -------------------------------------------------------------------------------- 1 | package ipvs 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "syscall" 7 | "testing" 8 | ) 9 | 10 | func Test_getIPFamily(t *testing.T) { 11 | testcases := []struct { 12 | name string 13 | address []byte 14 | expectedFamily uint16 15 | expectedErr error 16 | }{ 17 | { 18 | name: "16 byte IPv4 10.0.0.1", 19 | address: []byte{10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 20 | expectedFamily: syscall.AF_INET, 21 | expectedErr: nil, 22 | }, 23 | { 24 | name: "16 byte IPv6 2001:db8:3c4d:15::1a00", 25 | address: []byte{32, 1, 13, 184, 60, 77, 0, 21, 0, 0, 0, 0, 0, 0, 26, 0}, 26 | expectedFamily: syscall.AF_INET6, 27 | expectedErr: nil, 28 | }, 29 | { 30 | name: "zero address", 31 | address: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32 | expectedFamily: 0, 33 | expectedErr: errors.New("could not parse IP family from address data"), 34 | }, 35 | } 36 | 37 | for _, testcase := range testcases { 38 | testcase := testcase 39 | t.Run(testcase.name, func(t *testing.T) { 40 | family, err := getIPFamily(testcase.address) 41 | if !reflect.DeepEqual(err, testcase.expectedErr) { 42 | t.Logf("got err: %v", err) 43 | t.Logf("expected err: %v", testcase.expectedErr) 44 | t.Errorf("unexpected error") 45 | } 46 | 47 | if family != testcase.expectedFamily { 48 | t.Logf("got IP family: %v", family) 49 | t.Logf("expected IP family: %v", testcase.expectedFamily) 50 | t.Errorf("unexpected IP family") 51 | } 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /chart/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for chart. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | image: 6 | repository: ghcr.io/liorokman/proxmox-cloud-provider/proxomox-ccm 7 | pullPolicy: IfNotPresent 8 | # Overrides the image tag whose default is the chart appVersion. 9 | tag: "v0.0.2" 10 | 11 | # The mTLS configuration for lbManager 12 | lbmanager: 13 | hostname: 14 | port: "9999" 15 | tls: 16 | key: 17 | cert: 18 | caCert: 19 | 20 | proxmox: 21 | apiToken: 22 | user: 23 | apiURL: 24 | insecure: false 25 | caCert: 26 | timeout: 5 27 | 28 | 29 | # imagePullSecrets: [] 30 | # nameOverride: "" 31 | # fullnameOverride: "" 32 | 33 | serviceAccount: 34 | # Specifies whether a service account should be created 35 | create: true 36 | # Annotations to add to the service account 37 | annotations: {} 38 | # The name of the service account to use. 39 | # If not set and create is true, a name is generated using the fullname template 40 | #name: "" 41 | 42 | #podAnnotations: {} 43 | 44 | #podSecurityContext: {} 45 | # fsGroup: 2000 46 | 47 | #securityContext: {} 48 | # capabilities: 49 | # drop: 50 | # - ALL 51 | # readOnlyRootFilesystem: true 52 | # runAsNonRoot: true 53 | # runAsUser: 1000 54 | 55 | resources: {} 56 | # We usually recommend not to specify default resources and to leave this as a conscious 57 | # choice for the user. This also increases chances charts run on environments with little 58 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 59 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 60 | # limits: 61 | # cpu: 100m 62 | # memory: 128Mi 63 | # requests: 64 | # cpu: 100m 65 | # memory: 128Mi 66 | 67 | nodeSelector: {} 68 | 69 | -------------------------------------------------------------------------------- /internal/loadbalancer/loadbalancer.proto: -------------------------------------------------------------------------------- 1 | 2 | syntax = "proto3"; 3 | import "google/protobuf/empty.proto"; 4 | 5 | option go_package = "github.com/liorokman/proxmox-cloud-provider/internal/loadbalancer"; 6 | 7 | package loadbalancer; 8 | 9 | service LoadBalancer { 10 | // Get all information about all defined Load Balancers 11 | rpc GetLoadBalancers(google.protobuf.Empty) returns (stream LoadBalancerInformation); 12 | // Get information about a specific Load Balancer. If no such name exists, return 13 | // an empty structure 14 | rpc GetLoadBalancer(LoadBalancerName) returns (LoadBalancerInformation); 15 | 16 | rpc Create(CreateLoadBalancer) returns (LoadBalancerInformation); 17 | rpc Delete(LoadBalancerName) returns (Error); 18 | 19 | rpc AddTarget(AddTargetRequest) returns (Error); 20 | rpc DelTarget(DelTargetRequest) returns (Error); 21 | } 22 | 23 | enum Protocol { 24 | TCP = 0; 25 | UDP = 1; 26 | } 27 | 28 | message Error { 29 | uint32 Code = 1; 30 | string Message = 2; 31 | } 32 | 33 | message LoadBalancerName { 34 | string name = 1; 35 | } 36 | 37 | message LoadBalancerInformation { 38 | string name = 1; 39 | string ip_addr = 2; 40 | map targets = 3; 41 | } 42 | 43 | message CreateLoadBalancer { 44 | string name = 1; 45 | // If an ip_addr is requested, try to use it. Otherwise, an unused IP will be allocated. 46 | optional string ip_addr = 2; 47 | } 48 | 49 | message DelTargetRequest { 50 | string lb_name = 1; 51 | int32 srcPort = 2; 52 | Target target = 3; 53 | } 54 | 55 | message AddTargetRequest { 56 | string lb_name = 1; 57 | int32 srcPort = 2; 58 | Target target = 3; 59 | } 60 | 61 | message Target { 62 | Protocol protocol = 1; 63 | string dstIP = 2; 64 | int32 dstPort = 3; 65 | } 66 | 67 | message TargetList { 68 | repeated Target target = 1; 69 | } 70 | -------------------------------------------------------------------------------- /chart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "chart.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "chart.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "chart.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "chart.labels" -}} 37 | helm.sh/chart: {{ include "chart.chart" . }} 38 | {{ include "chart.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "chart.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "chart.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "chart.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "chart.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /chart/templates/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: {{ include "chart.fullname" . }} 5 | labels: 6 | {{- include "chart.labels" . | nindent 4 }} 7 | spec: 8 | selector: 9 | matchLabels: 10 | {{- include "chart.selectorLabels" . | nindent 6 }} 11 | template: 12 | metadata: 13 | {{- with .Values.podAnnotations }} 14 | annotations: 15 | {{- toYaml . | nindent 8 }} 16 | {{- end }} 17 | labels: 18 | {{- include "chart.selectorLabels" . | nindent 8 }} 19 | spec: 20 | {{- with .Values.imagePullSecrets }} 21 | imagePullSecrets: 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | volumes: 25 | - name: cloud-config 26 | configMap: 27 | name: {{ include "chart.fullname" . }}-cloud-config 28 | - name: proxmox-secrets 29 | secret: 30 | secretName: {{ include "chart.fullname" . }}-promox-lbctl 31 | serviceAccountName: {{ include "chart.serviceAccountName" . }} 32 | securityContext: 33 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 34 | containers: 35 | - name: {{ .Chart.Name }} 36 | securityContext: 37 | {{- toYaml .Values.securityContext | nindent 12 }} 38 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 39 | imagePullPolicy: {{ .Values.image.pullPolicy }} 40 | resources: 41 | {{- toYaml .Values.resources | nindent 12 }} 42 | command: 43 | - /proxmox-cloud-controller-manager 44 | - --cloud-provider=proxmox # Add your own cloud provider here! 45 | - --leader-elect=true 46 | - --use-service-account-credentials 47 | - --configure-cloud-routes=false 48 | - --cloud-config=/etc/proxmox/cloud-config.yaml 49 | volumeMounts: 50 | - name: cloud-config 51 | mountPath: /etc/proxmox 52 | - name: proxmox-secrets 53 | mountPath: /var/run/secrets/ccm 54 | tolerations: 55 | # this is required so CCM can bootstrap itself 56 | - key: node.cloudprovider.kubernetes.io/uninitialized 57 | value: "true" 58 | effect: NoSchedule 59 | # these tolerations are to have the daemonset runnable on control plane nodes 60 | # remove them if your control plane nodes should not run pods 61 | - key: node-role.kubernetes.io/control-plane 62 | operator: Exists 63 | effect: NoSchedule 64 | - key: node-role.kubernetes.io/master 65 | operator: Exists 66 | effect: NoSchedule 67 | nodeSelector: 68 | node-role.kubernetes.io/control-plane: "" 69 | {{- with .Values.nodeSelector }} 70 | {{- toYaml . | nindent 8 }} 71 | {{- end }} 72 | 73 | -------------------------------------------------------------------------------- /cmd/lbmanager/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "log" 8 | "net" 9 | "os" 10 | 11 | "github.com/knadh/koanf/parsers/yaml" 12 | "github.com/knadh/koanf/providers/file" 13 | "github.com/knadh/koanf/v2" 14 | "github.com/liorokman/proxmox-cloud-provider/internal/loadbalancer" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/credentials" 17 | ) 18 | 19 | var k = koanf.New(".") 20 | 21 | func init() { 22 | log.Printf("Loading configuration...") 23 | somethingLoaded := false 24 | err := loadConfigFile("/etc/lbmanager/lbmanager.yaml") 25 | if err == nil { 26 | somethingLoaded = true 27 | } else if !os.IsNotExist(err) { 28 | log.Fatalf("error loading config: %v", err) 29 | } 30 | 31 | err = loadConfigFile("lbmanager.yaml") 32 | if err == nil { 33 | somethingLoaded = true 34 | } else if !os.IsNotExist(err) { 35 | log.Fatalf("error loading local config: %v", err) 36 | } 37 | 38 | if !somethingLoaded { 39 | log.Fatalf("No configuration found. Cowardly refusing to continue.\n") 40 | } 41 | log.Printf("done\n") 42 | } 43 | 44 | func loadConfigFile(configFile string) error { 45 | mainConfigFile := file.Provider(configFile) 46 | if err := k.Load(mainConfigFile, yaml.Parser()); err != nil { 47 | return err 48 | } 49 | mainConfigFile.Watch(func(event any, err error) { 50 | if err != nil { 51 | log.Printf("watch error: %v\n", err) 52 | return 53 | } 54 | log.Println("config changed, reloading ...") 55 | tmpK := koanf.New(".") 56 | if err := tmpK.Load(mainConfigFile, yaml.Parser()); err != nil { 57 | log.Printf("error loading the new config: %v\n", err) 58 | return 59 | } 60 | k.Merge(tmpK) 61 | }) 62 | return nil 63 | } 64 | 65 | func newGRPCCreds(certFile, keyFile, caFile string) (credentials.TransportCredentials, error) { 66 | cert, err := tls.LoadX509KeyPair(certFile, keyFile) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | data, err := os.ReadFile(caFile) 72 | if err != nil { 73 | return nil, err 74 | } 75 | capool := x509.NewCertPool() 76 | if !capool.AppendCertsFromPEM(data) { 77 | return nil, err 78 | } 79 | return credentials.NewTLS(&tls.Config{ 80 | Certificates: []tls.Certificate{cert}, 81 | ClientCAs: capool, 82 | ClientAuth: tls.RequireAndVerifyClientCert, 83 | }), nil 84 | } 85 | 86 | func main() { 87 | 88 | creds, err := newGRPCCreds(k.MustString("grpc.auth.cert"), 89 | k.MustString("grpc.auth.key"), 90 | k.MustString("grpc.auth.ca")) 91 | if err != nil { 92 | log.Fatalf("error initializing TLS: %v", err) 93 | } 94 | 95 | lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", k.MustString("grpc.listen"), k.MustInt("grpc.port"))) 96 | if err != nil { 97 | log.Fatalf("failed listening: %v", err) 98 | } 99 | lbServer, err := loadbalancer.NewServer(k) 100 | if err != nil { 101 | log.Fatalf("failed starting the loadbalancer manager: %v", err) 102 | } 103 | defer lbServer.Close() 104 | if err := lbServer.Restore(); err != nil { 105 | log.Fatalf("failed restoring the loadbalancer configuration: %v", err) 106 | } 107 | opts := []grpc.ServerOption{ 108 | grpc.Creds(creds), 109 | } 110 | 111 | grpcServer := grpc.NewServer(opts...) 112 | loadbalancer.RegisterLoadBalancerServer(grpcServer, lbServer) 113 | log.Println("Listening...") 114 | grpcServer.Serve(lis) 115 | } 116 | -------------------------------------------------------------------------------- /internal/ipvs/ns/init_linux.go: -------------------------------------------------------------------------------- 1 | package ns 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/sirupsen/logrus" 12 | "github.com/vishvananda/netlink" 13 | "github.com/vishvananda/netns" 14 | "golang.org/x/sys/unix" 15 | ) 16 | 17 | var ( 18 | initNs netns.NsHandle 19 | initNl *netlink.Handle 20 | initOnce sync.Once 21 | ) 22 | 23 | // NetlinkSocketsTimeout represents the default timeout duration for the sockets 24 | const NetlinkSocketsTimeout = 3 * time.Second 25 | 26 | // Init initializes a new network namespace 27 | func Init() { 28 | var err error 29 | initNs, err = netns.Get() 30 | if err != nil { 31 | logrus.Errorf("could not get initial namespace: %v", err) 32 | } 33 | initNl, err = netlink.NewHandle(getSupportedNlFamilies()...) 34 | if err != nil { 35 | logrus.Errorf("could not create netlink handle on initial namespace: %v", err) 36 | } 37 | err = initNl.SetSocketTimeout(NetlinkSocketsTimeout) 38 | if err != nil { 39 | logrus.Warnf("Failed to set the timeout on the default netlink handle sockets: %v", err) 40 | } 41 | } 42 | 43 | // SetNamespace sets the initial namespace handler 44 | func SetNamespace() error { 45 | initOnce.Do(Init) 46 | if err := netns.Set(initNs); err != nil { 47 | linkInfo, linkErr := getLink() 48 | if linkErr != nil { 49 | linkInfo = linkErr.Error() 50 | } 51 | return fmt.Errorf("failed to set to initial namespace, %v, initns fd %d: %v", linkInfo, initNs, err) 52 | } 53 | return nil 54 | } 55 | 56 | // ParseHandlerInt transforms the namespace handler into an integer 57 | func ParseHandlerInt() int { 58 | return int(getHandler()) 59 | } 60 | 61 | // GetHandler returns the namespace handler 62 | func getHandler() netns.NsHandle { 63 | initOnce.Do(Init) 64 | return initNs 65 | } 66 | 67 | func getLink() (string, error) { 68 | return os.Readlink(fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())) 69 | } 70 | 71 | // NlHandle returns the netlink handler 72 | func NlHandle() *netlink.Handle { 73 | initOnce.Do(Init) 74 | return initNl 75 | } 76 | 77 | func getSupportedNlFamilies() []int { 78 | fams := []int{unix.NETLINK_ROUTE} 79 | // NETLINK_XFRM test 80 | if err := checkXfrmSocket(); err != nil { 81 | logrus.Warnf("Could not load necessary modules for IPSEC rules: %v", err) 82 | } else { 83 | fams = append(fams, unix.NETLINK_XFRM) 84 | } 85 | // NETLINK_NETFILTER test 86 | if err := loadNfConntrackModules(); err != nil { 87 | if checkNfSocket() != nil { 88 | logrus.Warnf("Could not load necessary modules for Conntrack: %v", err) 89 | } else { 90 | fams = append(fams, unix.NETLINK_NETFILTER) 91 | } 92 | } else { 93 | fams = append(fams, unix.NETLINK_NETFILTER) 94 | } 95 | 96 | return fams 97 | } 98 | 99 | // API check on required xfrm modules (xfrm_user, xfrm_algo) 100 | func checkXfrmSocket() error { 101 | fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_XFRM) 102 | if err != nil { 103 | return err 104 | } 105 | unix.Close(fd) 106 | return nil 107 | } 108 | 109 | func loadNfConntrackModules() error { 110 | if out, err := exec.Command("modprobe", "-va", "nf_conntrack").CombinedOutput(); err != nil { 111 | return fmt.Errorf("Running modprobe nf_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) 112 | } 113 | if out, err := exec.Command("modprobe", "-va", "nf_conntrack_netlink").CombinedOutput(); err != nil { 114 | return fmt.Errorf("Running modprobe nf_conntrack_netlink failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) 115 | } 116 | return nil 117 | } 118 | 119 | // API check on required nf_conntrack* modules (nf_conntrack, nf_conntrack_netlink) 120 | func checkNfSocket() error { 121 | fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_NETFILTER) 122 | if err != nil { 123 | return err 124 | } 125 | unix.Close(fd) 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // The external controller manager is responsible for running controller loops that 18 | // are cloud provider dependent. It uses the API to listen to new events on resources. 19 | 20 | // This file should be written by each cloud provider. 21 | // For a minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go 22 | // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go 23 | // The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use. 24 | 25 | package main 26 | 27 | import ( 28 | "os" 29 | 30 | "k8s.io/apimachinery/pkg/util/wait" 31 | cloudprovider "k8s.io/cloud-provider" 32 | "k8s.io/cloud-provider/app" 33 | cloudcontrollerconfig "k8s.io/cloud-provider/app/config" 34 | "k8s.io/cloud-provider/options" 35 | "k8s.io/component-base/cli" 36 | cliflag "k8s.io/component-base/cli/flag" 37 | _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins 38 | _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration 39 | "k8s.io/klog/v2" 40 | 41 | // For existing cloud providers, the option to import legacy providers is still available. 42 | // e.g. _"k8s.io/legacy-cloud-providers/" 43 | _ "github.com/liorokman/proxmox-cloud-provider/internal/proxmox" 44 | ) 45 | 46 | func main() { 47 | ccmOptions, err := options.NewCloudControllerManagerOptions() 48 | if err != nil { 49 | klog.Fatalf("unable to initialize command options: %v", err) 50 | } 51 | 52 | controllerInitializers := app.DefaultInitFuncConstructors 53 | // Here is an example to remove the controller which is not needed. 54 | // e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need. 55 | //delete(controllerInitializers, "cloud-node-lifecycle") 56 | 57 | // Here is an example to add an controller(NodeIpamController) which will be used by cloud provider 58 | // generate nodeIPAMConfig. Here is an sample code. 59 | // If you do not need additional controller, please ignore. 60 | 61 | /* 62 | nodeIpamController := nodeIPAMController{} 63 | nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration 64 | nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller")) 65 | 66 | controllerInitializers["nodeipam"] = app.ControllerInitFuncConstructor{ 67 | // "node-controller" is the shared identity of all node controllers, including node, node lifecycle, and node ipam. 68 | // See https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context. 69 | InitContext: app.ControllerInitContext{ 70 | ClientName: "node-controller", 71 | }, 72 | Constructor: nodeIpamController.StartNodeIpamControllerWrapper, 73 | } 74 | */ 75 | fss := cliflag.NamedFlagSets{} 76 | command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop) 77 | code := cli.Run(command) 78 | os.Exit(code) 79 | } 80 | 81 | func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface { 82 | cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider 83 | // initialize cloud provider with the cloud provider name and config file provided 84 | cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) 85 | if err != nil { 86 | klog.Fatalf("Cloud provider could not be initialized: %v", err) 87 | } 88 | if cloud == nil { 89 | klog.Fatalf("Cloud provider is nil") 90 | } 91 | 92 | if !cloud.HasClusterID() { 93 | if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { 94 | klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") 95 | } else { 96 | klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") 97 | } 98 | } 99 | 100 | return cloud 101 | } 102 | -------------------------------------------------------------------------------- /internal/ipvs/constants_linux.go: -------------------------------------------------------------------------------- 1 | package ipvs 2 | 3 | const ( 4 | genlCtrlID = 0x10 5 | ) 6 | 7 | // GENL control commands 8 | const ( 9 | genlCtrlCmdUnspec uint8 = iota 10 | genlCtrlCmdNewFamily 11 | genlCtrlCmdDelFamily 12 | genlCtrlCmdGetFamily 13 | ) 14 | 15 | // GENL family attributes 16 | const ( 17 | genlCtrlAttrUnspec int = iota 18 | genlCtrlAttrFamilyID 19 | genlCtrlAttrFamilyName 20 | ) 21 | 22 | // IPVS genl commands 23 | const ( 24 | ipvsCmdUnspec uint8 = iota 25 | ipvsCmdNewService 26 | ipvsCmdSetService 27 | ipvsCmdDelService 28 | ipvsCmdGetService 29 | ipvsCmdNewDest 30 | ipvsCmdSetDest 31 | ipvsCmdDelDest 32 | ipvsCmdGetDest 33 | ipvsCmdNewDaemon 34 | ipvsCmdDelDaemon 35 | ipvsCmdGetDaemon 36 | ipvsCmdSetConfig 37 | ipvsCmdGetConfig 38 | ipvsCmdSetInfo 39 | ipvsCmdGetInfo 40 | ipvsCmdZero 41 | ipvsCmdFlush 42 | ) 43 | 44 | // Attributes used in the first level of commands 45 | const ( 46 | ipvsCmdAttrUnspec int = iota 47 | ipvsCmdAttrService 48 | ipvsCmdAttrDest 49 | ipvsCmdAttrDaemon 50 | ipvsCmdAttrTimeoutTCP 51 | ipvsCmdAttrTimeoutTCPFin 52 | ipvsCmdAttrTimeoutUDP 53 | ) 54 | 55 | // Attributes used to describe a service. Used inside nested attribute 56 | // ipvsCmdAttrService 57 | const ( 58 | ipvsSvcAttrUnspec int = iota 59 | ipvsSvcAttrAddressFamily 60 | ipvsSvcAttrProtocol 61 | ipvsSvcAttrAddress 62 | ipvsSvcAttrPort 63 | ipvsSvcAttrFWMark 64 | ipvsSvcAttrSchedName 65 | ipvsSvcAttrFlags 66 | ipvsSvcAttrTimeout 67 | ipvsSvcAttrNetmask 68 | ipvsSvcAttrStats 69 | ipvsSvcAttrPEName 70 | ) 71 | 72 | // Attributes used to describe a destination (real server). Used 73 | // inside nested attribute ipvsCmdAttrDest. 74 | const ( 75 | ipvsDestAttrUnspec int = iota 76 | ipvsDestAttrAddress 77 | ipvsDestAttrPort 78 | ipvsDestAttrForwardingMethod 79 | ipvsDestAttrWeight 80 | ipvsDestAttrUpperThreshold 81 | ipvsDestAttrLowerThreshold 82 | ipvsDestAttrActiveConnections 83 | ipvsDestAttrInactiveConnections 84 | ipvsDestAttrPersistentConnections 85 | ipvsDestAttrStats 86 | ipvsDestAttrAddressFamily 87 | ) 88 | 89 | // IPVS Statistics constants 90 | 91 | const ( 92 | ipvsStatsUnspec int = iota 93 | ipvsStatsConns 94 | ipvsStatsPktsIn 95 | ipvsStatsPktsOut 96 | ipvsStatsBytesIn 97 | ipvsStatsBytesOut 98 | ipvsStatsCPS 99 | ipvsStatsPPSIn 100 | ipvsStatsPPSOut 101 | ipvsStatsBPSIn 102 | ipvsStatsBPSOut 103 | ) 104 | 105 | // Destination forwarding methods 106 | const ( 107 | // ConnectionFlagFwdmask indicates the mask in the connection 108 | // flags which is used by forwarding method bits. 109 | ConnectionFlagFwdMask = 0x0007 110 | 111 | // ConnectionFlagMasq is used for masquerade forwarding method. 112 | ConnectionFlagMasq = 0x0000 113 | 114 | // ConnectionFlagLocalNode is used for local node forwarding 115 | // method. 116 | ConnectionFlagLocalNode = 0x0001 117 | 118 | // ConnectionFlagTunnel is used for tunnel mode forwarding 119 | // method. 120 | ConnectionFlagTunnel = 0x0002 121 | 122 | // ConnectionFlagDirectRoute is used for direct routing 123 | // forwarding method. 124 | ConnectionFlagDirectRoute = 0x0003 125 | ) 126 | 127 | const ( 128 | // RoundRobin distributes jobs equally amongst the available 129 | // real servers. 130 | RoundRobin = "rr" 131 | 132 | // LeastConnection assigns more jobs to real servers with 133 | // fewer active jobs. 134 | LeastConnection = "lc" 135 | 136 | // DestinationHashing assigns jobs to servers through looking 137 | // up a statically assigned hash table by their destination IP 138 | // addresses. 139 | DestinationHashing = "dh" 140 | 141 | // SourceHashing assigns jobs to servers through looking up 142 | // a statically assigned hash table by their source IP 143 | // addresses. 144 | SourceHashing = "sh" 145 | 146 | // WeightedRoundRobin assigns jobs to real servers proportionally 147 | // to there real servers' weight. Servers with higher weights 148 | // receive new jobs first and get more jobs than servers 149 | // with lower weights. Servers with equal weights get 150 | // an equal distribution of new jobs 151 | WeightedRoundRobin = "wrr" 152 | 153 | // WeightedLeastConnection assigns more jobs to servers 154 | // with fewer jobs and relative to the real servers' weight 155 | WeightedLeastConnection = "wlc" 156 | ) 157 | 158 | const ( 159 | // ConnFwdMask is a mask for the fwd methods 160 | ConnFwdMask = 0x0007 161 | 162 | // ConnFwdMasq denotes forwarding via masquerading/NAT 163 | ConnFwdMasq = 0x0000 164 | 165 | // ConnFwdLocalNode denotes forwarding to a local node 166 | ConnFwdLocalNode = 0x0001 167 | 168 | // ConnFwdTunnel denotes forwarding via a tunnel 169 | ConnFwdTunnel = 0x0002 170 | 171 | // ConnFwdDirectRoute denotes forwarding via direct routing 172 | ConnFwdDirectRoute = 0x0003 173 | 174 | // ConnFwdBypass denotes forwarding while bypassing the cache 175 | ConnFwdBypass = 0x0004 176 | ) 177 | -------------------------------------------------------------------------------- /internal/ipam/simple.go: -------------------------------------------------------------------------------- 1 | package ipam 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "strings" 8 | 9 | "github.com/knadh/koanf/v2" 10 | "github.com/vishvananda/netlink" 11 | ) 12 | 13 | var _ IPAM = &SimpleIpam{} 14 | 15 | type Range struct { 16 | From net.IP 17 | To net.IP 18 | } 19 | 20 | type SimpleIpam struct { 21 | cidr *net.IPNet 22 | allowedRange Range 23 | link netlink.Link 24 | nlHandle *netlink.Handle 25 | } 26 | 27 | func New(config *koanf.Koanf, nlHandle *netlink.Handle, link netlink.Link) (IPAM, error) { 28 | _, cidr, err := net.ParseCIDR(config.MustString("loadbalancer.ipam.cidr")) 29 | if err != nil { 30 | return nil, err 31 | } 32 | dynamicRange := Range{ 33 | From: net.ParseIP(config.MustString("loadbalancer.ipam.dynamicRange.startAt")), 34 | To: net.ParseIP(config.MustString("loadbalancer.ipam.dynamicRange.endAt")), 35 | } 36 | if bytesCompare(dynamicRange.From, dynamicRange.To) >= 0 { 37 | return nil, fmt.Errorf("invalid dynamic range provided: (%s, %s)", dynamicRange.From, dynamicRange.To) 38 | } 39 | if !cidr.Contains(dynamicRange.From) || !cidr.Contains(dynamicRange.To) { 40 | return nil, fmt.Errorf("dynamic range (%s, %s) is not entirely contained in the cidr (%s)", 41 | dynamicRange.From, dynamicRange.To, cidr) 42 | } 43 | return &SimpleIpam{ 44 | cidr: cidr, 45 | allowedRange: dynamicRange, 46 | link: link, 47 | nlHandle: nlHandle, 48 | }, nil 49 | } 50 | 51 | func (r Range) Contains(ip net.IP) bool { 52 | if ip.IsUnspecified() { 53 | return false 54 | } 55 | if r.From.Equal(ip) || r.To.Equal(ip) { 56 | return true 57 | } 58 | 59 | return bytesCompare(ip, r.From) >= 0 && bytesCompare(ip, r.To) <= 0 60 | } 61 | 62 | func (s *SimpleIpam) Contains(ip net.IP) bool { 63 | return s.cidr.Contains(ip) 64 | } 65 | 66 | func (s *SimpleIpam) IsUsed(ip net.IP) bool { 67 | addrs, err := s.nlHandle.AddrList(s.link, netlink.FAMILY_ALL) 68 | if err != nil { 69 | return true 70 | } 71 | return isUsed(ip, addrs) 72 | } 73 | 74 | func (s *SimpleIpam) Allocate() (net.IP, error) { 75 | addrs, err := s.nlHandle.AddrList(s.link, netlink.FAMILY_ALL) 76 | if err != nil { 77 | return nil, err 78 | } 79 | ipTrie := newIPTrie() 80 | for _, addr := range addrs { 81 | ipTrie.Insert(addr.IP.To16()) 82 | } 83 | currCandidate := s.allowedRange.From 84 | for bytesCompare(currCandidate, s.allowedRange.To) <= 0 { 85 | if !ipTrie.Contains(currCandidate) { 86 | return currCandidate, nil 87 | } 88 | currCandidate = getNextIP(currCandidate) 89 | } 90 | return nil, fmt.Errorf("no free IP found in dynamic range") 91 | } 92 | 93 | func (s *SimpleIpam) Release(ip net.IP) error { 94 | // Not needed in the simple ipam, if the IP is released from the link 95 | // then it will be available to be allocated 96 | return nil 97 | } 98 | 99 | // Helper function to compare two net.IP addresses in byte form. 100 | // It returns -1 if a < b, 0 if a == b, and 1 if a > b. 101 | func bytesCompare(left, right net.IP) int { 102 | leftBytes := left.To16() 103 | rightBytes := right.To16() 104 | 105 | for i := 0; i < net.IPv6len; i++ { 106 | if leftBytes[i] < rightBytes[i] { 107 | return -1 108 | } else if leftBytes[i] > rightBytes[i] { 109 | return 1 110 | } 111 | } 112 | return 0 113 | } 114 | 115 | func isUsed(ip net.IP, addrs []netlink.Addr) bool { 116 | for _, addr := range addrs { 117 | if ip.Equal(addr.IP) { 118 | return true 119 | } 120 | } 121 | return false 122 | } 123 | 124 | func getNextIP(ip net.IP) net.IP { 125 | nextIP := make(net.IP, len(ip)) 126 | copy(nextIP, ip) 127 | 128 | for i := len(nextIP) - 1; i >= 0; i-- { 129 | nextIP[i]++ 130 | if nextIP[i] > 0 { 131 | break 132 | } 133 | } 134 | 135 | return nextIP 136 | } 137 | 138 | type ipNode struct { 139 | children map[byte]*ipNode 140 | isEnd bool 141 | } 142 | 143 | type ipTrie struct { 144 | root *ipNode 145 | } 146 | 147 | // newIPTrie creates a new IPTrie. 148 | func newIPTrie() *ipTrie { 149 | return &ipTrie{ 150 | root: &ipNode{ 151 | children: make(map[byte]*ipNode), 152 | }, 153 | } 154 | } 155 | 156 | func (n *ipNode) printToLog(lvl int) { 157 | if n == nil { 158 | return 159 | } 160 | for k, v := range n.children { 161 | log.Printf("%s %+v", strings.Repeat(" ", lvl), k) 162 | v.printToLog(lvl + 1) 163 | } 164 | } 165 | 166 | func (t *ipTrie) PrintToLog() { 167 | t.root.printToLog(0) 168 | } 169 | 170 | // Insert adds an IP address to the trie. 171 | func (t *ipTrie) Insert(ip net.IP) { 172 | node := t.root 173 | for _, b := range ip { 174 | if node.children[b] == nil { 175 | node.children[b] = &ipNode{ 176 | children: make(map[byte]*ipNode), 177 | } 178 | } 179 | node = node.children[b] 180 | } 181 | node.isEnd = true 182 | } 183 | 184 | // Contains checks if an IP address is present in the trie. 185 | func (t *ipTrie) Contains(ip net.IP) bool { 186 | node := t.root 187 | ip = ip.To16() 188 | for _, b := range ip { 189 | if node.children[b] == nil { 190 | return false 191 | } 192 | node = node.children[b] 193 | } 194 | return node.isEnd 195 | } 196 | -------------------------------------------------------------------------------- /scripts/installLB.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VMID=$1; shift 4 | PHASE=$1; shift 5 | 6 | urldecode () { 7 | : "${*//+/ }"; 8 | echo -e "${_//%/\\x}" 9 | } 10 | 11 | writeFile() { 12 | local filename=$1; shift 13 | 14 | local ct_notes=$(pct config $VMID | grep ^description | cut -f2- -d: ) 15 | ct_notes=$(urldecode $ct_notes) 16 | ext_ip=$(echo $ct_notes | jq -r .ip) 17 | ext_mask=$(echo $ct_notes | jq -r .mask) 18 | ext_end=$(echo $ct_notes | jq -r .end) 19 | ext_start=$(echo $ct_notes | jq -r .start) 20 | 21 | pct exec $VMID tee $filename << EOF 22 | #!/bin/bash 23 | 24 | # Install the necessary tools 25 | apt-get -y update 26 | apt-get -y upgrade 27 | apt-get -y install ipvsadm iptables jq bridge-utils wget golang-cfssl 28 | 29 | # Create the required network environment: 30 | # 1. Create a new namespace for the LB 31 | ip netns add LB 32 | 33 | # 2. Start the loopback in the new namespace 34 | ip netns exec LB ip link set lo up 35 | 36 | # 3. Move eth1 into it, copy the address it has into the new namespace 37 | ETH1_ADDR=\$(ip -j addr show dev eth1 | jq -r '.[0].addr_info[] | select( .family == "inet") | (.local) + "/" + (.prefixlen|tostring)') 38 | 39 | ip link set eth1 netns LB 40 | ip netns exec LB ip addr add \$ETH1_ADDR dev eth1 41 | ip netns exec LB ip link set eth1 up 42 | 43 | # 4. Create a new network bridge and a pair of veth devices 44 | brctl addbr br0 45 | ip link add veth1 type veth peer name external0 netns LB 46 | ip link set veth1 up 47 | brctl addif br0 veth1 48 | ip link set br0 up 49 | 50 | # 5. Move the ip address from eth0 to br0 and add eth0 to br0 51 | ETH0_ADDR=\$(ip -j addr show dev eth0 | jq -r '.[0].addr_info[] | select( .family == "inet") | (.local) + "/" + (.prefixlen|tostring)') 52 | ETH0_GW=\$(ip -j route | jq -r ' .[] | select( .dst == "default" ) | .gateway ' ) 53 | 54 | ip addr del \$ETH0_ADDR dev eth0 55 | brctl addif br0 eth0 56 | ip addr add \$ETH0_ADDR dev br0 57 | ip route add default via \$ETH0_GW 58 | 59 | # 6. Setup the bridged device in the LB network namespace 60 | ip netns exec LB ip link set external0 up 61 | ip netns exec LB ip addr add $ext_ip/$ext_mask dev external0 62 | ip netns exec LB ip route add default via \$ETH0_GW 63 | 64 | # 7. Setup ip forwarding in the LB namespace 65 | ip netns exec LB sysctl net.ipv4.ip_forward=1 66 | ip netns exec LB sysctl net.ipv4.vs.conntrack=1 67 | 68 | # 8. Setup SNAT so this container can serve as an internet gateway 69 | ip netns exec LB iptables -t nat -A POSTROUTING -m ipvs --destination \$ETH1_ADDR --vaddr $ext_ip -j MASQUERADE 70 | ip netns exec LB iptables -t nat -A POSTROUTING -o external0 -j SNAT --to-source $ext_ip 71 | 72 | # 9. Setup a CA for lbmanager 73 | plain_ip=\$(echo \$ETH0_ADDR | cut -f1 -d/) 74 | mkdir -p /var/lib/lbmanagerca 75 | if [ ! -f /var/lib/lbmanagerca/ca.pem ]; then 76 | pushd /var/lib/lbmanagerca > /dev/null 77 | # Certificate for lbmanager 78 | echo '{"key":{"algo":"rsa","size":2048},"names":[{"O":"HomeLab","CN":"Root LB CA"}]}' | \ 79 | cfssl genkey -initca - | \ 80 | cfssljson -bare ca 81 | echo "{\"hosts\": [\"\$plain_ip\",\"127.0.0.1\",\"localhost\"],\"key\":{\"algo\":\"rsa\",\"size\":2048},\"names\":[{\"O\":\"Homelab\",\"CN\":\"\$plain_ip\"}]}" | \ 82 | cfssl gencert -ca ca.pem -ca-key ca-key.pem - | \ 83 | cfssljson -bare lbmanager 84 | echo "{\"key\":{\"algo\":\"rsa\",\"size\":2048},\"names\":[{\"O\":\"Homelab\",\"CN\":\"lbctl\"}]}" | \ 85 | cfssl gencert -ca ca.pem -ca-key ca-key.pem - | \ 86 | cfssljson -bare lbctl 87 | popd > /dev/null 88 | fi 89 | 90 | # 10. Install lbmanager 91 | wget -q -O /usr/local/bin/lbmanager https://github.com/liorokman/proxmox-cloud-provider/releases/download/v0.0.2/lbmanager 92 | wget -q -O /usr/local/bin/lbctl https://github.com/liorokman/proxmox-cloud-provider/releases/download/v0.0.2/lbctl 93 | chmod +x /usr/local/bin/lbmanager /usr/local/bin/lbctl 94 | mkdir -p /etc/lbmanager 95 | cp /var/lib/lbmanagerca/lbmanager.pem /etc/lbmanager/cert.pem 96 | cp /var/lib/lbmanagerca/lbmanager-key.pem /etc/lbmanager/key.pem 97 | cp /var/lib/lbmanagerca/lbctl.pem /etc/lbmanager/lbctl.pem 98 | cp /var/lib/lbmanagerca/lbctl-key.pem /etc/lbmanager/lbctl-key.pem 99 | cp /var/lib/lbmanagerca/ca.pem /etc/lbmanager/ca.pem 100 | 101 | cat > /etc/lbmanager/lbmanager.yaml << END 102 | --- 103 | grpc: 104 | listen: 0.0.0.0 105 | port: 9999 106 | auth: 107 | cert: /etc/lbmanager/cert.pem 108 | key: /etc/lbmanager/key.pem 109 | ca: /etc/lbmanager/ca.pem 110 | dbDir: /var/lib/lbmanager 111 | loadbalancer: 112 | namespace: LB 113 | externalInterface: external0 114 | internalInterface: eth1 115 | ipam: 116 | cidr: $ext_ip/$ext_mask 117 | dynamicRange: 118 | startAt: $ext_start 119 | endAt: $ext_end 120 | END 121 | 122 | cat > /etc/lbmanager/lbctl.yaml << END 123 | --- 124 | addr: \$plain_ip:9999 125 | auth: 126 | clientKey: /etc/lbmanager/lbctl-key.pem 127 | clientCert: /etc/lbmanager/lbctl.pem 128 | caFile: /etc/lbmanager/ca.pem 129 | END 130 | 131 | cat > /etc/systemd/system/lbmanager.service << END 132 | [Unit] 133 | Description=LoadBalancer Manager 134 | ConditionPathExists=/etc/lbmanager/cert.pem 135 | 136 | [Service] 137 | ExecStart=/usr/local/bin/lbmanager 138 | Restart=on-failure 139 | 140 | [Install] 141 | Alias=lbmanager.service 142 | 143 | END 144 | 145 | systemctl daemon-reload 146 | systemctl start lbmanager 147 | 148 | 149 | EOF 150 | pct exec $VMID chmod +x $filename 151 | } 152 | 153 | case "$PHASE" in 154 | pre-start) 155 | ;; 156 | post-start) 157 | writeFile /install.sh 158 | pct exec $VMID /install.sh 159 | ;; 160 | pre-stop) 161 | ;; 162 | post-stop) 163 | ;; 164 | *) 165 | echo "Unknown phase $PHASE" 166 | exit 1 167 | ;; 168 | esac 169 | 170 | 171 | -------------------------------------------------------------------------------- /cmd/lbctl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "flag" 8 | "fmt" 9 | "io" 10 | "log" 11 | "os" 12 | 13 | "github.com/knadh/koanf/parsers/yaml" 14 | "github.com/knadh/koanf/providers/file" 15 | "github.com/knadh/koanf/v2" 16 | "google.golang.org/grpc" 17 | "google.golang.org/grpc/credentials" 18 | "google.golang.org/protobuf/types/known/emptypb" 19 | 20 | "github.com/liorokman/proxmox-cloud-provider/internal/loadbalancer" 21 | ) 22 | 23 | var k = koanf.New(".") 24 | 25 | var ( 26 | clientCert = flag.String("cert", "", "filename containing the client certificate") 27 | clientKey = flag.String("key", "", "filename containing the client certificate private key") 28 | caFile = flag.String("ca", "", "filename containing the CA that can verify the server") 29 | 30 | serverAddr = flag.String("addr", "", "server address") 31 | name = flag.String("name", "", "service name") 32 | op = flag.String("op", "addSrv", "addSrv, delSrv, addTgt, delTgt, list") 33 | 34 | service = flag.String("srv", "", "in {add,del}Srv - service ip, in {add,del}Tgt - target ip") 35 | protocol = flag.Bool("tcp", true, "true == TCP, false == UDP") 36 | dstPort = flag.Int("dport", 8080, "destination port") 37 | srcPort = flag.Int("sport", 8080, "source port") 38 | ) 39 | 40 | func init() { 41 | err := loadConfigFile("/etc/lbmanager/lbctl.yaml") 42 | if err != nil && !os.IsNotExist(err) { 43 | log.Fatalf("failed loading config: %v", err) 44 | } 45 | } 46 | 47 | func loadConfigFile(configFile string) error { 48 | f := file.Provider(configFile) 49 | if err := k.Load(f, yaml.Parser()); err != nil { 50 | return err 51 | } 52 | return nil 53 | } 54 | 55 | func loadKeypair() (credentials.TransportCredentials, error) { 56 | if *clientKey == "" { 57 | *clientKey = k.String("auth.clientKey") 58 | } 59 | if *clientCert == "" { 60 | *clientCert = k.String("auth.clientCert") 61 | } 62 | if *clientKey == "" { 63 | clientKey = clientCert 64 | } 65 | if *caFile == "" { 66 | *caFile = k.String("auth.caFile") 67 | } 68 | if *clientKey == "" || *clientCert == "" || *caFile == "" { 69 | return nil, fmt.Errorf("no mTLS configuration found") 70 | } 71 | cert, err := tls.LoadX509KeyPair(*clientCert, *clientKey) 72 | if err != nil { 73 | return nil, err 74 | } 75 | ca, err := os.ReadFile(*caFile) 76 | if err != nil { 77 | return nil, err 78 | } 79 | capool := x509.NewCertPool() 80 | if !capool.AppendCertsFromPEM(ca) { 81 | return nil, err 82 | } 83 | return credentials.NewTLS(&tls.Config{ 84 | Certificates: []tls.Certificate{cert}, 85 | RootCAs: capool, 86 | }), nil 87 | } 88 | 89 | func main() { 90 | flag.Parse() 91 | 92 | if *name == "" && *op != "list" { 93 | log.Fatal("no loadbalancer name provided.") 94 | } 95 | 96 | creds, err := loadKeypair() 97 | if err != nil { 98 | log.Fatalf("error loading mTLS configuration: %+v", err) 99 | } 100 | opts := []grpc.DialOption{ 101 | grpc.WithTransportCredentials(creds), 102 | } 103 | if *serverAddr == "" { 104 | *serverAddr = k.String("addr") 105 | } 106 | conn, err := grpc.Dial(*serverAddr, opts...) 107 | if err != nil { 108 | log.Fatalf("error connecting to lbmanager: %+v", err) 109 | } 110 | defer conn.Close() 111 | 112 | var p loadbalancer.Protocol = loadbalancer.Protocol_TCP 113 | if !*protocol { 114 | p = loadbalancer.Protocol_UDP 115 | } 116 | 117 | client := loadbalancer.NewLoadBalancerClient(conn) 118 | 119 | switch *op { 120 | case "addSrv": 121 | clb := &loadbalancer.CreateLoadBalancer{ 122 | Name: *name, 123 | IpAddr: service, 124 | } 125 | lbInfo, err := client.Create(context.Background(), clb) 126 | if err != nil { 127 | log.Fatalf("Error during create: %s", err.Error()) 128 | } 129 | log.Printf("LBInfo: %+v\n", lbInfo) 130 | 131 | case "delSrv": 132 | ret, err := client.Delete(context.Background(), &loadbalancer.LoadBalancerName{ 133 | Name: *name, 134 | }) 135 | if err != nil { 136 | log.Fatalf("Error during delete: %s", err.Error()) 137 | } 138 | if len(ret.Message) != 0 { 139 | log.Printf("Message from server: %s", ret.Message) 140 | } else { 141 | log.Printf("Success!") 142 | } 143 | case "addTgt": 144 | ret, err := client.AddTarget(context.Background(), &loadbalancer.AddTargetRequest{ 145 | LbName: *name, 146 | SrcPort: int32(*srcPort), 147 | Target: &loadbalancer.Target{ 148 | DstIP: *service, 149 | DstPort: int32(*dstPort), 150 | Protocol: p, 151 | }, 152 | }) 153 | if err != nil { 154 | log.Fatalf("Error adding destination: %s", err.Error()) 155 | } 156 | if len(ret.Message) != 0 { 157 | log.Printf("Message from server: %s", ret.Message) 158 | } else { 159 | log.Printf("Success!") 160 | } 161 | case "delTgt": 162 | ret, err := client.DelTarget(context.Background(), &loadbalancer.DelTargetRequest{ 163 | LbName: *name, 164 | SrcPort: int32(*srcPort), 165 | Target: &loadbalancer.Target{ 166 | DstIP: *service, 167 | DstPort: int32(*dstPort), 168 | Protocol: p, 169 | }, 170 | }) 171 | if err != nil { 172 | log.Fatalf("Error deleting destination: %s", err.Error()) 173 | } 174 | if len(ret.Message) != 0 { 175 | log.Printf("Message from server: %s", ret.Message) 176 | } else { 177 | log.Printf("Success!") 178 | } 179 | 180 | case "list": 181 | stream, err := client.GetLoadBalancers(context.Background(), &emptypb.Empty{}) 182 | if err != nil { 183 | log.Fatalf("Failed to list load balancers: %s", err.Error()) 184 | } 185 | for { 186 | lb, err := stream.Recv() 187 | if err == io.EOF { 188 | break 189 | } 190 | if err != nil { 191 | log.Fatalf("Error listing load balancers %v", err) 192 | } 193 | log.Println(lb) 194 | } 195 | default: 196 | log.Fatalf("Unknown operation %s", *op) 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /internal/proxmox/provider.go: -------------------------------------------------------------------------------- 1 | package proxmox 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | 11 | "google.golang.org/grpc" 12 | "google.golang.org/grpc/credentials" 13 | "gopkg.in/yaml.v2" 14 | cloudprovider "k8s.io/cloud-provider" 15 | 16 | "github.com/liorokman/proxmox-cloud-provider/internal/loadbalancer" 17 | ) 18 | 19 | // Provider implements the CCM interface 20 | type Provider struct { 21 | config ProviderConfig 22 | lbManager *LoadBalancer 23 | instances cloudprovider.InstancesV2 24 | } 25 | 26 | var _ cloudprovider.Interface = &Provider{} 27 | 28 | func init() { 29 | cloudprovider.RegisterCloudProvider("proxmox", New) 30 | } 31 | 32 | // ProviderConfig contains the configuration values required by the proxmox provider 33 | type ProviderConfig struct { 34 | Security Security `yaml:"security"` 35 | ManagerHost string `yaml:"managerHost"` 36 | ManagerPort int `yaml:"managerPort"` 37 | ClusterConfig ClusterConfig `yaml:"proxmoxConfig"` 38 | } 39 | 40 | // Security contains the mTLS configuration for connecting to the loadbalancer manager 41 | type Security struct { 42 | CertFile string `yaml:"cert"` 43 | KeyFile string `yaml:"key"` 44 | CAFile string `yaml:"ca"` 45 | } 46 | 47 | // ClusterConfig contains the proxmox cluster configuration information 48 | type ClusterConfig struct { 49 | URL string `yaml:"url"` 50 | Timeout int `yaml:"timeout"` 51 | APIToken string `yaml:"apiToken"` 52 | Username string `yaml:"username"` 53 | InsecureTLS bool `yaml:"insecureTLS"` 54 | CACert string `yaml:"caCert"` 55 | Debug bool `yaml:"debug"` 56 | } 57 | 58 | // New returns a new instance of the proxmox cloud provider 59 | func New(config io.Reader) (cloudprovider.Interface, error) { 60 | c, err := io.ReadAll(config) 61 | if err != nil { 62 | return nil, err 63 | } 64 | providerConfig := ProviderConfig{} 65 | err = yaml.Unmarshal(c, &providerConfig) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | return &Provider{ 71 | config: providerConfig, 72 | }, nil 73 | } 74 | 75 | // Initialize provides the cloud with a kubernetes client builder and may spawn goroutines 76 | // to perform housekeeping or run custom controllers specific to the cloud provider. 77 | // Any tasks started here should be cleaned up when the stop channel closes. 78 | func (p *Provider) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { 79 | 80 | instances, err := newInstances(p.config.ClusterConfig) 81 | if err != nil { 82 | log.Fatalf("failed to connect to proxmox: %s", err.Error()) 83 | } 84 | p.instances = instances 85 | 86 | creds, err := loadKeypair(p.config.Security) 87 | if err != nil { 88 | log.Fatalf("failed to load the lbmanager credentials: %s", err.Error()) 89 | } 90 | opts := []grpc.DialOption{ 91 | grpc.WithTransportCredentials(creds), 92 | } 93 | serverAddr := fmt.Sprintf("%s:%d", p.config.ManagerHost, p.config.ManagerPort) 94 | conn, err := grpc.Dial(serverAddr, opts...) 95 | if err != nil { 96 | log.Fatalf("failed to connect to the lbmanager: %s", err.Error()) 97 | } 98 | p.lbManager = &LoadBalancer{ 99 | client: loadbalancer.NewLoadBalancerClient(conn), 100 | } 101 | go func() { 102 | <-stop 103 | conn.Close() 104 | }() 105 | } 106 | 107 | // LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise. 108 | func (p *Provider) LoadBalancer() (cloudprovider.LoadBalancer, bool) { 109 | return p.lbManager, true 110 | } 111 | 112 | // Instances returns an instances interface. Also returns true if the interface is supported, false otherwise. 113 | func (p *Provider) Instances() (cloudprovider.Instances, bool) { 114 | return nil, false 115 | } 116 | 117 | // InstancesV2 is an implementation for instances and should only be implemented by external cloud providers. 118 | // Implementing InstancesV2 is behaviorally identical to Instances but is optimized to significantly reduce 119 | // API calls to the cloud provider when registering and syncing nodes. Implementation of this interface will 120 | // disable calls to the Zones interface. Also returns true if the interface is supported, false otherwise. 121 | func (p *Provider) InstancesV2() (cloudprovider.InstancesV2, bool) { 122 | return p.instances, true 123 | } 124 | 125 | // Zones returns a zones interface. Also returns true if the interface is supported, false otherwise. 126 | // DEPRECATED: Zones is deprecated in favor of retrieving zone/region information from InstancesV2. 127 | // This interface will not be called if InstancesV2 is enabled. 128 | func (p *Provider) Zones() (cloudprovider.Zones, bool) { 129 | return nil, false 130 | } 131 | 132 | // Clusters returns a clusters interface. Also returns true if the interface is supported, false otherwise. 133 | func (p *Provider) Clusters() (cloudprovider.Clusters, bool) { 134 | return nil, false 135 | } 136 | 137 | // Routes returns a routes interface along with whether the interface is supported. 138 | func (p *Provider) Routes() (cloudprovider.Routes, bool) { 139 | return nil, false 140 | } 141 | 142 | // ProviderName returns the cloud provider ID. 143 | func (p *Provider) ProviderName() string { 144 | return "proxmox" 145 | } 146 | 147 | // HasClusterID returns true if a ClusterID is required and set 148 | func (p *Provider) HasClusterID() bool { 149 | return true 150 | } 151 | 152 | func loadKeypair(security Security) (credentials.TransportCredentials, error) { 153 | if security.KeyFile == "" { 154 | security.KeyFile = security.CertFile 155 | } 156 | cert, err := tls.LoadX509KeyPair(security.CertFile, security.KeyFile) 157 | if err != nil { 158 | return nil, err 159 | } 160 | ca, err := os.ReadFile(security.CAFile) 161 | if err != nil { 162 | return nil, err 163 | } 164 | capool := x509.NewCertPool() 165 | if !capool.AppendCertsFromPEM(ca) { 166 | return nil, err 167 | } 168 | return credentials.NewTLS(&tls.Config{ 169 | Certificates: []tls.Certificate{cert}, 170 | RootCAs: capool, 171 | }), nil 172 | } 173 | 174 | func asPtr[T any](v T) *T { 175 | return &v 176 | } 177 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/liorokman/proxmox-cloud-provider 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/knadh/koanf/parsers/yaml v0.1.0 7 | github.com/knadh/koanf/providers/file v0.1.0 8 | github.com/knadh/koanf/v2 v2.0.1 9 | k8s.io/api v0.27.3 10 | k8s.io/apimachinery v0.27.3 11 | k8s.io/cloud-provider v0.27.3 12 | k8s.io/component-base v0.27.3 13 | k8s.io/klog/v2 v2.90.1 14 | ) 15 | 16 | require ( 17 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 18 | github.com/NYTimes/gziphandler v1.1.1 // indirect 19 | github.com/Telmate/proxmox-api-go v0.0.0-20230616173359-03f4e428f6c6 // indirect 20 | github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect 21 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect 22 | github.com/beorn7/perks v1.0.1 // indirect 23 | github.com/blang/semver/v4 v4.0.0 // indirect 24 | github.com/bwmarrin/snowflake v0.3.0 // indirect 25 | github.com/cenkalti/backoff/v4 v4.1.3 // indirect 26 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 27 | github.com/coreos/go-semver v0.3.0 // indirect 28 | github.com/coreos/go-systemd/v22 v22.4.0 // indirect 29 | github.com/davecgh/go-spew v1.1.1 // indirect 30 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 31 | github.com/evanphx/json-patch v4.12.0+incompatible // indirect 32 | github.com/felixge/httpsnoop v1.0.3 // indirect 33 | github.com/fsnotify/fsnotify v1.6.0 // indirect 34 | github.com/go-logr/logr v1.2.3 // indirect 35 | github.com/go-logr/stdr v1.2.2 // indirect 36 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 37 | github.com/go-openapi/jsonreference v0.20.1 // indirect 38 | github.com/go-openapi/swag v0.22.3 // indirect 39 | github.com/gofrs/flock v0.8.1 // indirect 40 | github.com/gogo/protobuf v1.3.2 // indirect 41 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 42 | github.com/golang/protobuf v1.5.3 // indirect 43 | github.com/google/cel-go v0.12.6 // indirect 44 | github.com/google/gnostic v0.5.7-v3refs // indirect 45 | github.com/google/go-cmp v0.5.9 // indirect 46 | github.com/google/gofuzz v1.1.0 // indirect 47 | github.com/google/uuid v1.3.0 // indirect 48 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 49 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect 50 | github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect 51 | github.com/imdario/mergo v0.3.6 // indirect 52 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 53 | github.com/josharian/intern v1.0.0 // indirect 54 | github.com/json-iterator/go v1.1.12 // indirect 55 | github.com/knadh/koanf/maps v0.1.1 // indirect 56 | github.com/mailru/easyjson v0.7.7 // indirect 57 | github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect 58 | github.com/mitchellh/copystructure v1.2.0 // indirect 59 | github.com/mitchellh/mapstructure v1.5.0 // indirect 60 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 61 | github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect 62 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 63 | github.com/modern-go/reflect2 v1.0.2 // indirect 64 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 65 | github.com/pkg/errors v0.9.1 // indirect 66 | github.com/prometheus/client_golang v1.14.0 // indirect 67 | github.com/prometheus/client_model v0.3.0 // indirect 68 | github.com/prometheus/common v0.37.0 // indirect 69 | github.com/prometheus/procfs v0.8.0 // indirect 70 | github.com/rosedblabs/go-immutable-radix/v2 v2.0.1-0.20230614125820-f2a7bc058c90 // indirect 71 | github.com/rosedblabs/rosedb/v2 v2.2.1 // indirect 72 | github.com/rosedblabs/wal v1.1.0 // indirect 73 | github.com/sirupsen/logrus v1.9.0 // indirect 74 | github.com/spf13/cobra v1.7.0 // indirect 75 | github.com/spf13/pflag v1.0.5 // indirect 76 | github.com/stoewer/go-strcase v1.2.0 // indirect 77 | github.com/vishvananda/netlink v1.1.0 // indirect 78 | github.com/vishvananda/netns v0.0.4 // indirect 79 | go.etcd.io/etcd/api/v3 v3.5.7 // indirect 80 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect 81 | go.etcd.io/etcd/client/v3 v3.5.7 // indirect 82 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect 83 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect 84 | go.opentelemetry.io/otel v1.10.0 // indirect 85 | go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect 86 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect 87 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect 88 | go.opentelemetry.io/otel/metric v0.31.0 // indirect 89 | go.opentelemetry.io/otel/sdk v1.10.0 // indirect 90 | go.opentelemetry.io/otel/trace v1.10.0 // indirect 91 | go.opentelemetry.io/proto/otlp v0.19.0 // indirect 92 | go.uber.org/atomic v1.7.0 // indirect 93 | go.uber.org/multierr v1.6.0 // indirect 94 | go.uber.org/zap v1.19.0 // indirect 95 | golang.org/x/crypto v0.1.0 // indirect 96 | golang.org/x/net v0.8.0 // indirect 97 | golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect 98 | golang.org/x/sync v0.1.0 // indirect 99 | golang.org/x/sys v0.10.0 // indirect 100 | golang.org/x/term v0.6.0 // indirect 101 | golang.org/x/text v0.8.0 // indirect 102 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect 103 | google.golang.org/appengine v1.6.7 // indirect 104 | google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect 105 | google.golang.org/grpc v1.51.0 // indirect 106 | google.golang.org/protobuf v1.28.1 // indirect 107 | gopkg.in/inf.v0 v0.9.1 // indirect 108 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 109 | gopkg.in/yaml.v2 v2.4.0 // indirect 110 | gopkg.in/yaml.v3 v3.0.1 // indirect 111 | k8s.io/apiserver v0.27.3 // indirect 112 | k8s.io/client-go v0.27.3 // indirect 113 | k8s.io/component-helpers v0.27.3 // indirect 114 | k8s.io/controller-manager v0.27.3 // indirect 115 | k8s.io/kms v0.27.3 // indirect 116 | k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect 117 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 118 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect 119 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 120 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 121 | sigs.k8s.io/yaml v1.3.0 // indirect 122 | ) 123 | -------------------------------------------------------------------------------- /internal/ipvs/ipvs_linux.go: -------------------------------------------------------------------------------- 1 | package ipvs 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "time" 7 | 8 | "github.com/vishvananda/netlink/nl" 9 | "github.com/vishvananda/netns" 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | const ( 14 | netlinkRecvSocketsTimeout = 3 * time.Second 15 | netlinkSendSocketTimeout = 30 * time.Second 16 | ) 17 | 18 | // Service defines an IPVS service in its entirety. 19 | type Service struct { 20 | // Virtual service address. 21 | Address net.IP 22 | Protocol uint16 23 | Port uint16 24 | FWMark uint32 // Firewall mark of the service. 25 | 26 | // Virtual service options. 27 | SchedName string 28 | Flags uint32 29 | Timeout uint32 30 | Netmask uint32 31 | AddressFamily uint16 32 | PEName string 33 | Stats SvcStats 34 | } 35 | 36 | // SvcStats defines an IPVS service statistics 37 | type SvcStats struct { 38 | Connections uint32 39 | PacketsIn uint32 40 | PacketsOut uint32 41 | BytesIn uint64 42 | BytesOut uint64 43 | CPS uint32 44 | BPSOut uint32 45 | PPSIn uint32 46 | PPSOut uint32 47 | BPSIn uint32 48 | } 49 | 50 | // Destination defines an IPVS destination (real server) in its 51 | // entirety. 52 | type Destination struct { 53 | Address net.IP 54 | Port uint16 55 | Weight int 56 | ConnectionFlags uint32 57 | AddressFamily uint16 58 | UpperThreshold uint32 59 | LowerThreshold uint32 60 | ActiveConnections int 61 | InactiveConnections int 62 | Stats DstStats 63 | } 64 | 65 | // DstStats defines IPVS destination (real server) statistics 66 | type DstStats SvcStats 67 | 68 | // Config defines IPVS timeout configuration 69 | type Config struct { 70 | TimeoutTCP time.Duration 71 | TimeoutTCPFin time.Duration 72 | TimeoutUDP time.Duration 73 | } 74 | 75 | // Handle provides a namespace specific ipvs handle to program ipvs 76 | // rules. 77 | type Handle struct { 78 | seq uint32 79 | sock *nl.NetlinkSocket 80 | } 81 | 82 | // New provides a new ipvs handle in the namespace pointed to by the 83 | // passed path. It will return a valid handle or an error in case an 84 | // error occurred while creating the handle. 85 | func New(path string) (*Handle, error) { 86 | n := netns.None() 87 | if path != "" { 88 | var err error 89 | n, err = netns.GetFromPath(path) 90 | if err != nil { 91 | return nil, err 92 | } 93 | } 94 | defer n.Close() 95 | return NewInNamespace(n) 96 | } 97 | 98 | // NewInNamespace provides a new ipvs handle in the namespace provided 99 | // as a parameter. It will return a valid handle or an error in case an 100 | // error occurred while creating the handle. 101 | func NewInNamespace(n netns.NsHandle) (*Handle, error) { 102 | setup() 103 | 104 | sock, err := nl.GetNetlinkSocketAt(n, netns.None(), unix.NETLINK_GENERIC) 105 | if err != nil { 106 | return nil, err 107 | } 108 | // Add operation timeout to avoid deadlocks 109 | tv := unix.NsecToTimeval(netlinkSendSocketTimeout.Nanoseconds()) 110 | if err := sock.SetSendTimeout(&tv); err != nil { 111 | return nil, err 112 | } 113 | tv = unix.NsecToTimeval(netlinkRecvSocketsTimeout.Nanoseconds()) 114 | if err := sock.SetReceiveTimeout(&tv); err != nil { 115 | return nil, err 116 | } 117 | 118 | return &Handle{sock: sock}, nil 119 | } 120 | 121 | // Close closes the ipvs handle. The handle is invalid after Close 122 | // returns. 123 | func (i *Handle) Close() { 124 | if i.sock != nil { 125 | i.sock.Close() 126 | } 127 | } 128 | 129 | // NewService creates a new ipvs service in the passed handle. 130 | func (i *Handle) NewService(s *Service) error { 131 | return i.doCmd(s, nil, ipvsCmdNewService) 132 | } 133 | 134 | // IsServicePresent queries for the ipvs service in the passed handle. 135 | func (i *Handle) IsServicePresent(s *Service) bool { 136 | return nil == i.doCmd(s, nil, ipvsCmdGetService) 137 | } 138 | 139 | // UpdateService updates an already existing service in the passed 140 | // handle. 141 | func (i *Handle) UpdateService(s *Service) error { 142 | return i.doCmd(s, nil, ipvsCmdSetService) 143 | } 144 | 145 | // DelService deletes an already existing service in the passed 146 | // handle. 147 | func (i *Handle) DelService(s *Service) error { 148 | return i.doCmd(s, nil, ipvsCmdDelService) 149 | } 150 | 151 | // Flush deletes all existing services in the passed 152 | // handle. 153 | func (i *Handle) Flush() error { 154 | _, err := i.doCmdWithoutAttr(ipvsCmdFlush) 155 | return err 156 | } 157 | 158 | // NewDestination creates a new real server in the passed ipvs 159 | // service which should already be existing in the passed handle. 160 | func (i *Handle) NewDestination(s *Service, d *Destination) error { 161 | return i.doCmd(s, d, ipvsCmdNewDest) 162 | } 163 | 164 | // UpdateDestination updates an already existing real server in the 165 | // passed ipvs service in the passed handle. 166 | func (i *Handle) UpdateDestination(s *Service, d *Destination) error { 167 | return i.doCmd(s, d, ipvsCmdSetDest) 168 | } 169 | 170 | // DelDestination deletes an already existing real server in the 171 | // passed ipvs service in the passed handle. 172 | func (i *Handle) DelDestination(s *Service, d *Destination) error { 173 | return i.doCmd(s, d, ipvsCmdDelDest) 174 | } 175 | 176 | // GetServices returns an array of services configured on the Node 177 | func (i *Handle) GetServices() ([]*Service, error) { 178 | return i.doGetServicesCmd(nil) 179 | } 180 | 181 | // GetDestinations returns an array of Destinations configured for this Service 182 | func (i *Handle) GetDestinations(s *Service) ([]*Destination, error) { 183 | return i.doGetDestinationsCmd(s, nil) 184 | } 185 | 186 | // GetService gets details of a specific IPVS services, useful in updating statisics etc., 187 | func (i *Handle) GetService(s *Service) (*Service, error) { 188 | res, err := i.doGetServicesCmd(s) 189 | if err != nil { 190 | return nil, err 191 | } 192 | 193 | // We are looking for exactly one service otherwise error out 194 | if len(res) != 1 { 195 | return nil, fmt.Errorf("Expected only one service obtained=%d", len(res)) 196 | } 197 | 198 | return res[0], nil 199 | } 200 | 201 | // GetConfig returns the current timeout configuration 202 | func (i *Handle) GetConfig() (*Config, error) { 203 | return i.doGetConfigCmd() 204 | } 205 | 206 | // SetConfig set the current timeout configuration. 0: no change 207 | func (i *Handle) SetConfig(c *Config) error { 208 | return i.doSetConfigCmd(c) 209 | } 210 | -------------------------------------------------------------------------------- /internal/proxmox/instance.go: -------------------------------------------------------------------------------- 1 | package proxmox 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "fmt" 8 | "log" 9 | "net" 10 | "net/url" 11 | "os" 12 | "strconv" 13 | "strings" 14 | 15 | "github.com/Telmate/proxmox-api-go/proxmox" 16 | v1 "k8s.io/api/core/v1" 17 | cloudprovider "k8s.io/cloud-provider" 18 | ) 19 | 20 | /* 21 | 22 | ProviderID for Proxmox based kubernetes nodes uses the following schema: 23 | 24 | proxmox://[cluster name]/[node name]/[vm id] 25 | 26 | */ 27 | 28 | // Instances implements the cloudprovider.InstancesV2 interface 29 | type Instances struct { 30 | client *proxmox.Client 31 | } 32 | 33 | var _ cloudprovider.InstancesV2 = &Instances{} 34 | 35 | func newInstances(config ClusterConfig) (cloudprovider.InstancesV2, error) { 36 | tls := &tls.Config{} 37 | if config.InsecureTLS { 38 | tls.InsecureSkipVerify = true 39 | } else { 40 | if config.CACert != "" { 41 | capool := x509.NewCertPool() 42 | data, err := os.ReadFile(config.CACert) 43 | if err != nil { 44 | return nil, err 45 | } 46 | if !capool.AppendCertsFromPEM(data) { 47 | return nil, fmt.Errorf("Failed to load ca certificate") 48 | } 49 | tls.RootCAs = capool 50 | } 51 | } 52 | c, err := proxmox.NewClient(config.URL, nil, "", tls, "", config.Timeout) 53 | if err != nil { 54 | return nil, err 55 | } 56 | username, err := os.ReadFile(config.Username) 57 | if err != nil { 58 | return nil, err 59 | } 60 | apiToken, err := os.ReadFile(config.APIToken) 61 | if err != nil { 62 | return nil, err 63 | } 64 | proxmox.Debug = &config.Debug 65 | c.SetAPIToken(strings.Trim(string(username), "\n\r \t"), 66 | strings.Trim(string(apiToken), "\n\r \t")) 67 | if _, err := c.GetVersion(); err != nil { 68 | return nil, fmt.Errorf("bad username password: (%+v) %w", c, err) 69 | } 70 | return &Instances{ 71 | client: c, 72 | }, nil 73 | 74 | } 75 | 76 | // InstanceExists returns true if the instance for the given node exists according to the cloud provider. 77 | // Use the node.name or node.spec.providerID field to find the node in the cloud provider. 78 | func (i *Instances) InstanceExists(ctx context.Context, node *v1.Node) (bool, error) { 79 | log.Printf("InstanceExists called for node %s ", node.Name) 80 | vmRef, err := i.getVMRef(ctx, node) 81 | if err != nil { 82 | return false, nil 83 | } 84 | _, err = i.getVMInfo(vmRef) 85 | if err != nil { 86 | return false, nil 87 | } 88 | return true, nil 89 | } 90 | 91 | // InstanceShutdown returns true if the instance is shutdown according to the cloud provider. 92 | // Use the node.name or node.spec.providerID field to find the node in the cloud provider. 93 | func (i *Instances) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) { 94 | log.Printf("InstanceShutdown called for node %s ", node.Name) 95 | vmRef, err := i.getVMRef(ctx, node) 96 | if err != nil { 97 | return false, err 98 | } 99 | vmInfo, err := i.getVMInfo(vmRef) 100 | if err != nil { 101 | return false, err 102 | } 103 | if status, found := vmInfo["status"]; found { 104 | return status != "running", nil 105 | } 106 | return false, fmt.Errorf("unexpected vm info - no status field returned: %+v", vmInfo) 107 | } 108 | 109 | // InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are 110 | // translated into specific fields and labels in the Node object on registration. 111 | // Implementations should always check node.spec.providerID first when trying to discover the instance 112 | // for a given node. In cases where node.spec.providerID is empty, implementations can use other 113 | // properties of the node like its name, labels and annotations. 114 | func (i *Instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) { 115 | log.Printf("InstanceMetadata called for node %s ", node.Name) 116 | vmRef, err := i.getVMRef(ctx, node) 117 | if err != nil { 118 | return nil, err 119 | } 120 | vmInfo, err := i.getVMInfo(vmRef) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | vmConfig, err := i.client.GetVmConfig(vmRef) 126 | if err != nil { 127 | return nil, err 128 | } 129 | 130 | var nodeName string 131 | var vmType string 132 | var found bool 133 | if nodeName, found = vmInfo["node"].(string); !found { 134 | log.Printf("info doesn't contain the node name") 135 | } 136 | if vmType, found = vmInfo["type"].(string); !found { 137 | log.Printf("info doesn't contain the vm type") 138 | } 139 | 140 | retval := &cloudprovider.InstanceMetadata{ 141 | ProviderID: getProviderID(vmInfo), 142 | InstanceType: vmType, 143 | NodeAddresses: getAddresses(vmConfig), 144 | Zone: nodeName, 145 | } 146 | 147 | log.Printf("Metadata returns %+v", retval) 148 | return retval, nil 149 | } 150 | 151 | func getAddresses(config map[string]any) []v1.NodeAddress { 152 | retval := []v1.NodeAddress{} 153 | for k, v := range config { 154 | if strings.HasPrefix(k, "ipconfig") { 155 | // Sometimes there's an ipconfig? entry without a net? entry. 156 | if _, found := config["net"+k[8:]]; !found { 157 | continue 158 | } 159 | parts := strings.Split(v.(string), ",") 160 | for _, currPart := range parts { 161 | if before, after, found := strings.Cut(currPart, "="); found { 162 | if before == "ip" { 163 | ip, _, err := net.ParseCIDR(after) 164 | if err != nil { 165 | log.Printf("unexpected ip address %s: %s", currPart, err.Error()) 166 | continue 167 | } 168 | retval = append(retval, v1.NodeAddress{ 169 | Type: v1.NodeInternalIP, 170 | Address: ip.String(), 171 | }) 172 | // already found the IP, no need to go on in this configuration key 173 | break 174 | } 175 | } 176 | } 177 | } else if k == "name" { 178 | retval = append(retval, v1.NodeAddress{ 179 | Type: v1.NodeHostName, 180 | Address: v.(string), 181 | }) 182 | } 183 | } 184 | return retval 185 | } 186 | 187 | func getProviderID(info map[string]any) string { 188 | // TODO: the current proxmox-go-api library doesn't provide an API to retrieve the cluster name. Ignore it for now. 189 | nodeName := "" 190 | vmid := 0.0 191 | found := false 192 | if nodeName, found = info["node"].(string); !found { 193 | log.Printf("info doesn't contain the node name") 194 | } 195 | if vmid, found = info["vmid"].(float64); !found { 196 | log.Printf("info doesn't contain the vmid") 197 | } 198 | return fmt.Sprintf("proxmox://%s/%s/%d", "", nodeName, int(vmid)) 199 | } 200 | 201 | func (i *Instances) getVMRef(ctx context.Context, node *v1.Node) (*proxmox.VmRef, error) { 202 | if i.client == nil { 203 | return nil, fmt.Errorf("not connected to any proxmox cluster") 204 | } 205 | var vmRef *proxmox.VmRef 206 | if node.Spec.ProviderID != "" { 207 | if providerURL, err := url.Parse(node.Spec.ProviderID); err == nil { 208 | parts := strings.Split(providerURL.Path, "/") 209 | // The expected format is "/node name/vm id" 210 | if len(parts) == 3 { 211 | if id, err := strconv.Atoi(parts[2]); err == nil { 212 | vmRef = proxmox.NewVmRef(id) 213 | } else { 214 | log.Printf("ignoring invalid providerID: %s, id part is not numeric: %s", node.Spec.ProviderID, err.Error()) 215 | } 216 | } else { 217 | log.Printf("ignoring invalid providerID: %s, path should be '/'", node.Spec.ProviderID) 218 | } 219 | } else { 220 | log.Printf("ignoring invalid providerID: %s", node.Spec.ProviderID) 221 | } 222 | } 223 | if vmRef == nil { 224 | // Failed to identify the node using ProviderID, try using the node.name 225 | var err error 226 | vmRef, err = i.client.GetVmRefByName(node.Name) 227 | if err != nil { 228 | return nil, err 229 | } 230 | } 231 | return vmRef, nil 232 | } 233 | 234 | func (i *Instances) getVMInfo(vmRef *proxmox.VmRef) (map[string]any, error) { 235 | if vmRef != nil { 236 | vmInfo, err := i.client.GetVmInfo(vmRef) 237 | if err != nil { 238 | return nil, err 239 | } 240 | return vmInfo, nil 241 | } 242 | return nil, fmt.Errorf("empty vmref provided") 243 | } 244 | -------------------------------------------------------------------------------- /internal/proxmox/loadbalancer.go: -------------------------------------------------------------------------------- 1 | package proxmox 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | "github.com/liorokman/proxmox-cloud-provider/internal/loadbalancer" 9 | v1 "k8s.io/api/core/v1" 10 | cloudprovider "k8s.io/cloud-provider" 11 | ) 12 | 13 | type LoadBalancer struct { 14 | client loadbalancer.LoadBalancerClient 15 | } 16 | 17 | var _ cloudprovider.LoadBalancer = &LoadBalancer{} 18 | 19 | // GetLoadBalancer returns whether the specified load balancer exists, and 20 | // if so, what its status is. 21 | // Implementations must treat the *v1.Service parameter as read-only and not modify it. 22 | // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager 23 | func (p *LoadBalancer) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) { 24 | portStatus := []v1.PortStatus{} 25 | lbName := p.GetLoadBalancerName(ctx, clusterName, service) 26 | lbInfo, err := p.client.GetLoadBalancer(ctx, &loadbalancer.LoadBalancerName{ 27 | Name: lbName, 28 | }) 29 | if err != nil || lbInfo.Name == "" { 30 | return nil, false, err 31 | } 32 | for _, port := range service.Spec.Ports { 33 | currPortError := "" 34 | if tgt, found := lbInfo.Targets[port.Port]; !found || len(tgt.Target) == 0 { 35 | currPortError = fmt.Sprintf("Port %d is not mapped", port.Port) 36 | } 37 | portStatus = append(portStatus, v1.PortStatus{ 38 | Port: port.Port, 39 | Protocol: "TCP", 40 | Error: &currPortError, 41 | }) 42 | } 43 | retval := &v1.LoadBalancerStatus{ 44 | Ingress: []v1.LoadBalancerIngress{ 45 | { 46 | IP: lbInfo.IpAddr, 47 | Ports: portStatus, 48 | }, 49 | }, 50 | } 51 | 52 | return retval, lbInfo.IpAddr != "", nil 53 | } 54 | 55 | // GetLoadBalancerName returns the name of the load balancer. Implementations must treat the 56 | // *v1.Service parameter as read-only and not modify it. 57 | func (p *LoadBalancer) GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string { 58 | return fmt.Sprintf("%s%%%s%%%s", 59 | clusterName, 60 | service.Namespace, 61 | service.Name) 62 | } 63 | 64 | // EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer 65 | // Implementations must treat the *v1.Service and *v1.Node 66 | // parameters as read-only and not modify them. 67 | // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager 68 | func (p *LoadBalancer) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { 69 | log.Printf("EnsureLoadBalancer called for service %+v", service) 70 | name := p.GetLoadBalancerName(ctx, clusterName, service) 71 | lbRequest := &loadbalancer.CreateLoadBalancer{ 72 | Name: name, 73 | } 74 | if service.Spec.LoadBalancerIP != "" { 75 | lbRequest.IpAddr = &service.Spec.LoadBalancerIP 76 | } 77 | lbinfo, err := p.client.GetLoadBalancer(ctx, &loadbalancer.LoadBalancerName{ 78 | Name: name, 79 | }) 80 | if lbinfo == nil || lbinfo.Name == "" { 81 | // No such loadbalancer exists 82 | lbinfo, err = p.client.Create(ctx, lbRequest) 83 | log.Printf("Calling create for %+v", lbRequest) 84 | if err != nil { 85 | log.Printf("Create failed: %+v", err) 86 | return nil, err 87 | } 88 | } 89 | log.Printf("LBInfo: %+v", lbinfo) 90 | ports := p.handleMappings(ctx, lbinfo, service, nodes) 91 | return &v1.LoadBalancerStatus{ 92 | Ingress: []v1.LoadBalancerIngress{ 93 | { 94 | IP: lbinfo.IpAddr, 95 | Ports: ports, 96 | }, 97 | }, 98 | }, nil 99 | } 100 | 101 | // UpdateLoadBalancer updates hosts under the specified load balancer. 102 | // Implementations must treat the *v1.Service and *v1.Node 103 | // parameters as read-only and not modify them. 104 | // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager 105 | func (p *LoadBalancer) UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error { 106 | log.Printf("UpdateLoadBalancer called: %+v +%v %+v", clusterName, service, nodes) 107 | name := p.GetLoadBalancerName(ctx, clusterName, service) 108 | lbinfo, err := p.client.GetLoadBalancer(ctx, &loadbalancer.LoadBalancerName{ 109 | Name: name, 110 | }) 111 | if lbinfo.Name == "" { 112 | // No such loadbalancer exists 113 | return err 114 | } 115 | log.Printf("LBInfo: %+v", lbinfo) 116 | p.handleMappings(ctx, lbinfo, service, nodes) 117 | return nil 118 | } 119 | 120 | func (p *LoadBalancer) handleMappings(ctx context.Context, lbinfo *loadbalancer.LoadBalancerInformation, service *v1.Service, nodes []*v1.Node) []v1.PortStatus { 121 | 122 | currentMappings := map[targetKey]int{} 123 | for port, targetList := range lbinfo.Targets { 124 | if targetList.Target != nil { 125 | for i, v := range targetList.Target { 126 | currentMappings[newTargetKey(v, port)] = i 127 | } 128 | } 129 | } 130 | log.Printf("Current mappings: %+v", currentMappings) 131 | ports := []v1.PortStatus{} 132 | for _, port := range service.Spec.Ports { 133 | if toLBProtocol(port.Protocol) < 0 { 134 | ports = append(ports, v1.PortStatus{ 135 | Port: port.Port, 136 | Protocol: port.Protocol, 137 | Error: asPtr(fmt.Sprintf("unsupported protocol: %s", port.Protocol)), 138 | }) 139 | continue 140 | } 141 | atr := &loadbalancer.AddTargetRequest{ 142 | LbName: lbinfo.Name, 143 | SrcPort: port.Port, 144 | } 145 | for _, currNode := range nodes { 146 | nodeIP := "" 147 | for _, currAddr := range currNode.Status.Addresses { 148 | if currAddr.Type == v1.NodeInternalIP { 149 | nodeIP = currAddr.Address 150 | break 151 | } 152 | } 153 | if nodeIP == "" { 154 | continue 155 | } 156 | target := &loadbalancer.Target{ 157 | DstIP: nodeIP, 158 | DstPort: port.NodePort, 159 | Protocol: toLBProtocol(port.Protocol), 160 | } 161 | currTargetKey := newTargetKey(target, port.Port) 162 | if _, found := currentMappings[currTargetKey]; !found { 163 | atr.Target = target 164 | stat := v1.PortStatus{ 165 | Port: port.Port, 166 | Protocol: port.Protocol, 167 | } 168 | log.Printf("Adding a mapping for srcPort: %d %+v", port.Port, target) 169 | code, err := p.client.AddTarget(ctx, atr) 170 | log.Printf("err: %+v code: %+v", err, code) 171 | if err != nil { 172 | stat.Error = asPtr(err.Error()) 173 | } else if code.Code != loadbalancer.ErrSuccess { 174 | stat.Error = asPtr(code.Message) 175 | } 176 | ports = append(ports, stat) 177 | } else { 178 | log.Printf("Mapping already exists for %+v", target) 179 | ports = append(ports, v1.PortStatus{ 180 | Port: port.Port, 181 | Protocol: port.Protocol, 182 | }) 183 | // Mark this protocol/ip/port combination as handled 184 | delete(currentMappings, currTargetKey) 185 | } 186 | } 187 | // Whatever is left in currentMappings should be removed 188 | for k := range currentMappings { 189 | log.Printf("removing mapping for %+v", k) 190 | dtr := &loadbalancer.DelTargetRequest{ 191 | LbName: lbinfo.Name, 192 | SrcPort: k.SrcPort, 193 | Target: k.toTarget(), 194 | } 195 | code, err := p.client.DelTarget(ctx, dtr) 196 | if err != nil { 197 | log.Printf("Failed to remove unused mapping %s: %s", k.String(), err) 198 | } else if code.Code != loadbalancer.ErrSuccess { 199 | log.Printf("Failed to remove unused mapping %s: (%d) %s", k.String(), code.Code, code.Message) 200 | } 201 | } 202 | } 203 | log.Printf("finished handling mappings. Ports: %+v", ports) 204 | return ports 205 | } 206 | 207 | // EnsureLoadBalancerDeleted deletes the specified load balancer if it 208 | // exists, returning nil if the load balancer specified either didn't exist or 209 | // was successfully deleted. 210 | // This construction is useful because many cloud providers' load balancers 211 | // have multiple underlying components, meaning a Get could say that the LB 212 | // doesn't exist even if some part of it is still laying around. 213 | // Implementations must treat the *v1.Service parameter as read-only and not modify it. 214 | // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager 215 | func (p *LoadBalancer) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { 216 | log.Printf("EnsureLoadBalancerDeleted called: %+v +%v", clusterName, service) 217 | name := p.GetLoadBalancerName(ctx, clusterName, service) 218 | code, err := p.client.Delete(ctx, &loadbalancer.LoadBalancerName{ 219 | Name: name, 220 | }) 221 | if err != nil { 222 | return err 223 | } else if code.Code != loadbalancer.ErrSuccess && code.Code != loadbalancer.ErrNoSuchLB { 224 | return fmt.Errorf("Failed to delete loadbalancer %s: (%d) %s", name, code.Code, code.Message) 225 | } 226 | return nil 227 | } 228 | 229 | type targetKey struct { 230 | IP string 231 | DstPort int32 232 | SrcPort int32 233 | Protocol loadbalancer.Protocol 234 | } 235 | 236 | func (t *targetKey) String() string { 237 | return fmt.Sprintf("(%d) %d->%s:%d", t.Protocol, t.SrcPort, t.IP, t.DstPort) 238 | } 239 | 240 | func newTargetKey(t *loadbalancer.Target, srcPort int32) targetKey { 241 | return targetKey{ 242 | IP: t.DstIP, 243 | DstPort: t.DstPort, 244 | SrcPort: srcPort, 245 | Protocol: t.Protocol, 246 | } 247 | } 248 | 249 | func (t *targetKey) toTarget() *loadbalancer.Target { 250 | return &loadbalancer.Target{ 251 | Protocol: t.Protocol, 252 | DstIP: t.IP, 253 | DstPort: t.DstPort, 254 | } 255 | } 256 | 257 | func toLBProtocol(p v1.Protocol) loadbalancer.Protocol { 258 | switch p { 259 | case v1.ProtocolTCP: 260 | return loadbalancer.Protocol_TCP 261 | case v1.ProtocolUDP: 262 | return loadbalancer.Protocol_UDP 263 | } 264 | return -1 265 | } 266 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Proxmox Cloud Controller Provider 3 | 4 | This projects provides all that is needed to create Kubernetes clusters on 5 | Proxmox where the Kubernetes clusters hosts are in their own Proxmox isolated 6 | SDN VNet. 7 | 8 | The Proxmox SDN functionality allows creating an isolated network for the 9 | Kubernetes nodes, and this project provides the glue required to add a load balancer 10 | and allow Kubernetes to configure it when `LoadBalancer` services are created. 11 | 12 | 13 | # Proxmox Setup 14 | 15 | The following steps are required to prepare Proxmox for Kubernetes clusters 16 | using this cloud controller provider: 17 | 18 | 1. Enable the SDN functionality 19 | 20 | Full documentation is available [here](https://pve.proxmox.com/pve-docs/chapter-pvesdn.html). 21 | 22 | 1. Install the `jq` utility on Proxmox 23 | 24 | This utility is used by the hookscript for creating the loadbalancer. 25 | 26 | ```bash 27 | apt install jq 28 | ``` 29 | 30 | 1. Configure an SDN zone for Kubernetes clusters 31 | 32 | Create a VXLAN or VLAN zone that will contain VNets for your Kubernetes clusters. 33 | 34 | Make sure that the zone is configured correctly to allow for all Proxmox 35 | nodes in the cluster to communicate. 36 | 37 | 1. Create an SDN VNet in the newly created zone. 38 | 39 | This VNet will contain subnets for each Kubernetes cluster. After the VNet 40 | is created, the Proxmox nodes will contain a bridge device called by the 41 | name given to the VNet. For the rest of this README, the VNet name is 42 | assumed to be `k8s`. 43 | 44 | 1. Prepare a Proxmox Storage that allows Snippets. 45 | 46 | This storage will be used for storing the load-balancer hookscript. Called 47 | `$SNIPPET_STORAGE` in this README. 48 | 49 | 1. Prepare a Proxmox Storage for VMs and LXC Containers. 50 | 51 | Called `$STORAGE` in this README. 52 | 53 | 1. Prepare an API token for the Proxmox Cloud Provider 54 | 55 | > Note: The provided example command creates a very powerful API token, which is probably overkill. A future README will specify the exact privileges required. 56 | 57 | ```bash 58 | pveum user token add root@pam ccm -privsep=0 59 | ``` 60 | 61 | Set `PROXMOX_API_TOKEN` to the generated token, and set `PROXMOX_API_USERNAME` 62 | to the correct username. In this example, the username would be 63 | `root@pam!ccm` 64 | 65 | # Creating a Kubernetes Cluster 66 | 67 | ## Required Information 68 | 69 | 1. Choose a range of VM IDs to be used for the Kubernetes 70 | cluster. In this README, all Kubernetes related VMs and container will be in 71 | the range 1100-1200 . 72 | 73 | 1. Allocate a management IP address in the main network bridge (`vmbr0`) for 74 | the loadbalancer. 75 | 76 | This IP will be used for connecting to the load-balancer container for 77 | management purposes. Called `$LB_MNG_IP` in this README. 78 | 79 | 1. Allocate an IP address in the main network bridge (`vmbr0`) for the 80 | Kubernetes API server. Called `$LB_EXT_IP` in this README. 81 | 82 | 1. Find out the main network's CIDR and gateway. Respectivly `$EXT_CIDR` and 83 | `$EXT_GW`. 84 | 85 | 1. Reserve a range of IPs for the loadbalancer to allocate when services are 86 | created. Called `$LB_EXT_START` and `$LB_EXT_END` respectively. 87 | 88 | For example, the following allocates 150 IPs for the loadbalancer 89 | ```bash 90 | LB_EXT_START=192.168.50.50 91 | LB_EXT_END=192.168.50.200 92 | ``` 93 | 94 | ## Setup a LoadBalancer LXC Container 95 | 96 | Each Kubernetes cluster requires a lightweight LXC container that will be used 97 | to configure an `IPVS` based load-balancer. 98 | 99 | > Note: The current implementation supports only one load balancer container - this 100 | will be fixed in future versions of this project. 101 | 102 | 1. Create a subnet for the new cluster in the `k8s` VNet. Enable SNAT, and set 103 | the gateway IP to the first IP in the subnet. 104 | 105 | For this README, the the subnet is `192.168.50.0/24`, and the gateway IP is 106 | `192.168.50.1`. 107 | 108 | 1. Download the latest Debian template. 109 | 110 | As of this writing, that template was version 12.0-1. 111 | 112 | 1. Create the container 113 | 114 | The container is also the default gateway to the main network bridge, so the 115 | `net1` interface should use the same IP that was allocated for the gateway 116 | in the Kubernetes nodes subnet. 117 | 118 | Make sure to run the container as a privileged container - this is required 119 | for `ipvs` to work from inside the container. 120 | 121 | ```bash 122 | DESCRIPTION="{\"ip\":\"$LB_EXT_IP\",\"mask\":24,\"start\":\"$LB_EXT_START\",\"end\":\"$LB_EXT_END\"}" 123 | pct create 1100 local:vztmpl/debian-12-standard_12.0-1_amd64.tar.zst \ 124 | -unprivileged 0 \ 125 | -cores 2 -swap 0 -memory 512 -hostname k8slb1 \ 126 | -net0 name=eth0,bridge=vmbr0,firewall=1,ip=$LB_EXT_IP/$EXT_CIDR,gw=$EXT_GW,type=veth \ 127 | -net1 name=eth1,bridge=k8s,firewall=1,ip=192.168.50.1/24,type=veth \ 128 | -ostype debian -features nesting=1 \ 129 | -password=$PASSWORD -storage $STORAGE \ 130 | -description $DESCRIPTION 131 | ``` 132 | 133 | 1. Copy the configuration hookscript `scripts/installLB.sh` to 134 | `$SNIPPET_STORAGE/snippets` on the Proxmox node where the load balancer 135 | container was created. 136 | 137 | 1. Enable the hookscript for the load balancer container. 138 | 139 | This script installs all the needed utilities on the container, and 140 | configures the network correctly. 141 | 142 | ```bash 143 | pct set 1100 -hookscript $SNIPPET_STORAGE:snippets/installLB.sh 144 | ``` 145 | 146 | 1. Start the container. 147 | 148 | ```bash 149 | pct start 1100 150 | ``` 151 | 152 | ## Prepare a VM Template for Kubernetes Nodes 153 | 154 | Follow the instructions in [this](https://github.com/liorokman/packer-Debian#use-in-proxmox) 155 | Github repository to prepare a template that can be used for creating Kubernetes nodes. 156 | 157 | A pre-built image is available in the [releases](https://github.com/liorokman/proxmox-cloud-provider/releases/tag/v0.0.2) section of this project with `kubeadm`, `kubelet`, and `kubectl` versions 1.27.4 . 158 | 159 | This README assumes that the template's ID is `$K8S_TEMPLATE_ID` 160 | 161 | ## Create and start the first node 162 | 163 | 1. Clone the template 164 | 165 | Choose an unused ID for the new VM. This section of the README assumes `$K8S_NEW_ID` is 166 | set to to the ID. 167 | 168 | ```bash 169 | qm clone $K8S_TEMPLATE_ID $K8S_NEW_ID -full -name k8s-master1 -storage $STORAGE 170 | ``` 171 | 172 | 1. Update the cloud-init parameters 173 | 1. Set a valid ssh key for the `debian` user 174 | 175 | ```bash 176 | qm set $K8S_NEW_ID -sshkeys /path/to/ssh/public/key/file 177 | ``` 178 | 179 | 1. Set a valid IP address. The default gateway should be the loadbalancer's 180 | ip in the `k8s` subnet: `$LB_EXT_IP` 181 | 182 | ```bash 183 | qm set $K8S_NEW_ID -ipconfig0 ip=192.168.50.3/24,gw=$LB_EXT_IP 184 | ``` 185 | 1. Regenerate the cloud-init volume 186 | 187 | ```bash 188 | qm cloudinit update $K8S_NEW_ID 189 | ``` 190 | 191 | 1. Resize the disk to at least 20Gb 192 | 193 | ```bash 194 | qm disk resize $K8S_NEW_ID virtio0 20G 195 | ``` 196 | 197 | 1. Start the vm 198 | 199 | ```bash 200 | qm start $K8S_NEW_ID 201 | ``` 202 | 203 | ## Finalize the loadbalancer configuration 204 | 205 | Enter the loadbalancer container and: 206 | 207 | 1. Configure a loadbalancer that reaches the master node's API server. 208 | 209 | ```bash 210 | pct enter 1100 211 | /usr/local/bin/lbctl -op addSrv -name apiserver -srv $PUBLIC_K8S_IP 212 | /usr/local/bin/lbctl -op addTgt -name apiserver -srv $NODE_INTERNAL_IP -sport 6443 -dport 6443 213 | ``` 214 | 215 | 1. Prepare credentials for the Proxmox CCM provider 216 | 217 | Reuse the credentials created for `lbctl` by copying `/etc/lbmanager/lbctl.pem`, 218 | `/etc/lbmanager/lbctl-key.pem`, and `/etc/lbmanager/ca.pem` to the master node being prepared. 219 | 220 | ## Configure Kubeadm and create the cluster 221 | 222 | Login as `debian` to the new Kubernetes master node, and move to `root` using `sudo -s`. 223 | 224 | 1. Initialize Kubernetes and install a CNI 225 | 226 | Add any parameters required for your CNI of choice. 227 | 228 | ```bash 229 | kubeadm init --control-plane-endpoint ${K8S_PUBLIC_APISERVER_DNSNAME}:6443 --upload-certs 230 | 231 | # Install your choice of CNI 232 | 233 | ``` 234 | 235 | 1. Install the Proxmox CCM provider 236 | 237 | Prepare the following files: 238 | 239 | * From the kubernetes master node: `/etc/kubernetes/admin.conf` 240 | 241 | The credentials to connect to Kubernetes, needed to allow Helm to connect to 242 | Kubernetes. 243 | 244 | * From your Proxmox host: `/etc/pve/pve-root-ca.pem` 245 | 246 | The certificate authority that signs the Proxmox API certificate, needed to 247 | connect to the Proxmox API. 248 | 249 | * From your loadbalancer: `/etc/lbmanager/lbctl.pem`, `/etc/lbmanager/lbctl-key.pem`, and `/etc/lbmanager/ca.pem` 250 | 251 | The mTLS credentials needed to connect to the load-balancer manager 252 | 253 | Install the Proxmox cloud provider with the following helm command: 254 | ```bash 255 | helm install --kubeconfig admin.conf proxmox-ccm ./chart \ 256 | --namespace kube-system \ 257 | --set lbmanager.hostname=$LB_MNG_IP \ 258 | --set-file lbmanager.tls.key=lbctl-key.pem \ 259 | --set-file lbmanager.tls.cert=lbctl.pem \ 260 | --set-file lbmanager.tls.caCert=ca.pem \ 261 | --set proxmox.apiToken=$PROXMOX_API_TOKEN \ 262 | --set proxmox.user=$PROXMOX_API_USERNAME \ 263 | --set proxmox.apiURL=https://$PROXMOX_HOSTNAME:8006/api2/json \ 264 | --set-file proxmox.caCert=pve-root-ca.pem 265 | ``` 266 | 267 | 268 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /internal/ipvs/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /internal/ipvs/ipvs_linux_test.go: -------------------------------------------------------------------------------- 1 | package ipvs 2 | 3 | import ( 4 | "net" 5 | "reflect" 6 | "runtime" 7 | "syscall" 8 | "testing" 9 | "time" 10 | 11 | "github.com/vishvananda/netlink" 12 | "github.com/vishvananda/netlink/nl" 13 | "golang.org/x/sys/unix" 14 | 15 | "github.com/liorokman/proxmox-cloud-provider/internal/ipvs/ns" 16 | ) 17 | 18 | var ( 19 | schedMethods = []string{ 20 | RoundRobin, 21 | LeastConnection, 22 | DestinationHashing, 23 | SourceHashing, 24 | WeightedLeastConnection, 25 | WeightedRoundRobin, 26 | } 27 | 28 | protocols = []string{ 29 | "TCP", 30 | "UDP", 31 | "FWM", 32 | } 33 | 34 | fwdMethods = []uint32{ 35 | ConnectionFlagMasq, 36 | ConnectionFlagTunnel, 37 | ConnectionFlagDirectRoute, 38 | } 39 | 40 | fwdMethodStrings = []string{ 41 | "Masq", 42 | "Tunnel", 43 | "Route", 44 | } 45 | ) 46 | 47 | func lookupFwMethod(fwMethod uint32) string { 48 | switch fwMethod { 49 | case ConnectionFlagMasq: 50 | return fwdMethodStrings[0] 51 | case ConnectionFlagTunnel: 52 | return fwdMethodStrings[1] 53 | case ConnectionFlagDirectRoute: 54 | return fwdMethodStrings[2] 55 | } 56 | return "" 57 | } 58 | 59 | func checkDestination(t *testing.T, i *Handle, s *Service, d *Destination, checkPresent bool) { 60 | var dstFound bool 61 | 62 | dstArray, err := i.GetDestinations(s) 63 | if err != nil { 64 | t.Fatalf("Failed to get destination; %v", err) 65 | } 66 | 67 | for _, dst := range dstArray { 68 | if dst.Address.Equal(d.Address) && dst.Port == d.Port && 69 | lookupFwMethod(dst.ConnectionFlags) == lookupFwMethod(d.ConnectionFlags) && 70 | dst.AddressFamily == d.AddressFamily { 71 | dstFound = true 72 | break 73 | } 74 | } 75 | 76 | switch checkPresent { 77 | case true: // The test expects the service to be present 78 | if !dstFound { 79 | t.Fatalf("Did not find the service %s in ipvs output", d.Address.String()) 80 | } 81 | case false: // The test expects that the service should not be present 82 | if dstFound { 83 | t.Fatalf("Did not find the destination %s fwdMethod %s in ipvs output", d.Address.String(), lookupFwMethod(d.ConnectionFlags)) 84 | } 85 | } 86 | } 87 | 88 | func checkService(t *testing.T, i *Handle, s *Service, checkPresent bool) { 89 | svcArray, err := i.GetServices() 90 | if err != nil { 91 | t.Fatalf("Failed to get service; %v", err) 92 | } 93 | 94 | var svcFound bool 95 | 96 | for _, svc := range svcArray { 97 | if svc.Protocol == s.Protocol && svc.Address.String() == s.Address.String() && svc.Port == s.Port { 98 | svcFound = true 99 | break 100 | } 101 | } 102 | 103 | switch checkPresent { 104 | case true: // The test expects the service to be present 105 | if !svcFound { 106 | t.Fatalf("Did not find the service %s in ipvs output", s.Address.String()) 107 | } 108 | case false: // The test expects that the service should not be present 109 | if svcFound { 110 | t.Fatalf("Did not expect the service %s in ipvs output", s.Address.String()) 111 | } 112 | } 113 | } 114 | 115 | func TestGetFamily(t *testing.T) { 116 | id, err := getIPVSFamily() 117 | if err != nil { 118 | t.Fatal("Failed to get IPVS family:", err) 119 | } 120 | if id == 0 { 121 | t.Error("IPVS family was 0") 122 | } 123 | } 124 | 125 | func TestService(t *testing.T) { 126 | defer setupTestOSContext(t)() 127 | 128 | i, err := New("") 129 | if err != nil { 130 | t.Fatal("Failed to create IPVS handle:", err) 131 | } 132 | 133 | for _, protocol := range protocols { 134 | for _, schedMethod := range schedMethods { 135 | testDatas := []struct { 136 | AddressFamily uint16 137 | IP string 138 | Netmask uint32 139 | }{ 140 | { 141 | AddressFamily: nl.FAMILY_V4, 142 | IP: "1.2.3.4", 143 | Netmask: 0xFFFFFFFF, 144 | }, { 145 | AddressFamily: nl.FAMILY_V6, 146 | IP: "2001:db8:3c4d:15::1a00", 147 | Netmask: 128, 148 | }, 149 | } 150 | for _, td := range testDatas { 151 | s := Service{ 152 | AddressFamily: td.AddressFamily, 153 | SchedName: schedMethod, 154 | } 155 | 156 | switch protocol { 157 | case "FWM": 158 | s.FWMark = 1234 159 | s.Netmask = td.Netmask 160 | case "TCP": 161 | s.Protocol = unix.IPPROTO_TCP 162 | s.Port = 80 163 | s.Address = net.ParseIP(td.IP) 164 | s.Netmask = td.Netmask 165 | case "UDP": 166 | s.Protocol = unix.IPPROTO_UDP 167 | s.Port = 53 168 | s.Address = net.ParseIP(td.IP) 169 | s.Netmask = td.Netmask 170 | } 171 | 172 | err := i.NewService(&s) 173 | if err != nil { 174 | t.Fatal("Failed to create service:", err) 175 | } 176 | checkService(t, i, &s, true) 177 | for _, updateSchedMethod := range schedMethods { 178 | if updateSchedMethod == schedMethod { 179 | continue 180 | } 181 | 182 | s.SchedName = updateSchedMethod 183 | err = i.UpdateService(&s) 184 | if err != nil { 185 | t.Fatal("Failed to update service:", err) 186 | } 187 | checkService(t, i, &s, true) 188 | 189 | scopy, err := i.GetService(&s) 190 | if err != nil { 191 | t.Fatal("Failed to get service:", err) 192 | } 193 | if expected := (*scopy).Address.String(); expected != s.Address.String() { 194 | t.Errorf("expected: %v, got: %v", expected, s.Address.String()) 195 | } 196 | if expected := (*scopy).Port; expected != s.Port { 197 | t.Errorf("expected: %v, got: %v", expected, s.Port) 198 | } 199 | if expected := (*scopy).Protocol; expected != s.Protocol { 200 | t.Errorf("expected: %v, got: %v", expected, s.Protocol) 201 | } 202 | } 203 | 204 | err = i.DelService(&s) 205 | if err != nil { 206 | t.Fatal("Failed to delete service:", err) 207 | } 208 | checkService(t, i, &s, false) 209 | } 210 | } 211 | } 212 | 213 | svcs := []Service{ 214 | { 215 | AddressFamily: nl.FAMILY_V4, 216 | SchedName: RoundRobin, 217 | Protocol: unix.IPPROTO_TCP, 218 | Port: 80, 219 | Address: net.ParseIP("10.20.30.40"), 220 | Netmask: 0xFFFFFFFF, 221 | }, 222 | { 223 | AddressFamily: nl.FAMILY_V4, 224 | SchedName: LeastConnection, 225 | Protocol: unix.IPPROTO_UDP, 226 | Port: 8080, 227 | Address: net.ParseIP("10.20.30.41"), 228 | Netmask: 0xFFFFFFFF, 229 | }, 230 | } 231 | // Create services for testing flush 232 | for _, svc := range svcs { 233 | if !i.IsServicePresent(&svc) { 234 | err = i.NewService(&svc) 235 | if err != nil { 236 | t.Fatal("Failed to create service:", err) 237 | } 238 | checkService(t, i, &svc, true) 239 | } else { 240 | t.Errorf("svc: %v exists", svc) 241 | } 242 | } 243 | err = i.Flush() 244 | if err != nil { 245 | t.Fatal("Failed to flush:", err) 246 | } 247 | got, err := i.GetServices() 248 | if err != nil { 249 | t.Fatal("Failed to get service:", err) 250 | } 251 | if len(got) != 0 { 252 | t.Errorf("Unexpected services after flush") 253 | } 254 | } 255 | 256 | func createDummyInterface(t *testing.T) { 257 | dummy := &netlink.Dummy{ 258 | LinkAttrs: netlink.LinkAttrs{ 259 | Name: "dummy", 260 | }, 261 | } 262 | 263 | err := netlink.LinkAdd(dummy) 264 | if err != nil { 265 | t.Fatal("Failed to add link:", err) 266 | } 267 | 268 | dummyLink, err := netlink.LinkByName("dummy") 269 | if err != nil { 270 | t.Fatal("Failed to get dummy link:", err) 271 | } 272 | 273 | ip, ipNet, err := net.ParseCIDR("10.1.1.1/24") 274 | if err != nil { 275 | t.Fatal("Failed to parse CIDR:", err) 276 | } 277 | 278 | ipNet.IP = ip 279 | 280 | ipAddr := &netlink.Addr{IPNet: ipNet, Label: ""} 281 | err = netlink.AddrAdd(dummyLink, ipAddr) 282 | if err != nil { 283 | t.Fatal("Failed to add IP address:", err) 284 | } 285 | } 286 | 287 | func TestDestination(t *testing.T) { 288 | defer setupTestOSContext(t)() 289 | 290 | createDummyInterface(t) 291 | i, err := New("") 292 | if err != nil { 293 | t.Fatal("Failed to create IPVS handle:", err) 294 | } 295 | 296 | for _, protocol := range protocols { 297 | testDatas := []struct { 298 | AddressFamily uint16 299 | IP string 300 | Netmask uint32 301 | Destinations []string 302 | }{ 303 | { 304 | AddressFamily: nl.FAMILY_V4, 305 | IP: "1.2.3.4", 306 | Netmask: 0xFFFFFFFF, 307 | Destinations: []string{"10.1.1.2", "10.1.1.3", "10.1.1.4"}, 308 | }, { 309 | AddressFamily: nl.FAMILY_V6, 310 | IP: "2001:db8:3c4d:15::1a00", 311 | Netmask: 128, 312 | Destinations: []string{"2001:db8:3c4d:15::1a2b", "2001:db8:3c4d:15::1a2c", "2001:db8:3c4d:15::1a2d"}, 313 | }, 314 | } 315 | for _, td := range testDatas { 316 | s := Service{ 317 | AddressFamily: td.AddressFamily, 318 | SchedName: RoundRobin, 319 | } 320 | 321 | switch protocol { 322 | case "FWM": 323 | s.FWMark = 1234 324 | s.Netmask = td.Netmask 325 | case "TCP": 326 | s.Protocol = unix.IPPROTO_TCP 327 | s.Port = 80 328 | s.Address = net.ParseIP(td.IP) 329 | s.Netmask = td.Netmask 330 | case "UDP": 331 | s.Protocol = unix.IPPROTO_UDP 332 | s.Port = 53 333 | s.Address = net.ParseIP(td.IP) 334 | s.Netmask = td.Netmask 335 | } 336 | 337 | err := i.NewService(&s) 338 | if err != nil { 339 | t.Fatal("Failed to create service:", err) 340 | } 341 | checkService(t, i, &s, true) 342 | 343 | s.SchedName = "" 344 | for _, fwdMethod := range fwdMethods { 345 | destinations := make([]Destination, 0) 346 | for _, ip := range td.Destinations { 347 | d := Destination{ 348 | AddressFamily: td.AddressFamily, 349 | Address: net.ParseIP(ip), 350 | Port: 5000, 351 | Weight: 1, 352 | ConnectionFlags: fwdMethod, 353 | } 354 | destinations = append(destinations, d) 355 | err := i.NewDestination(&s, &d) 356 | if err != nil { 357 | t.Fatal("Failed to create destination:", err) 358 | } 359 | checkDestination(t, i, &s, &d, true) 360 | } 361 | 362 | for _, updateFwdMethod := range fwdMethods { 363 | if updateFwdMethod == fwdMethod { 364 | continue 365 | } 366 | for _, d := range destinations { 367 | d.ConnectionFlags = updateFwdMethod 368 | err = i.UpdateDestination(&s, &d) 369 | if err != nil { 370 | t.Fatal("Failed to update destination:", err) 371 | } 372 | checkDestination(t, i, &s, &d, true) 373 | } 374 | } 375 | for _, d := range destinations { 376 | err = i.DelDestination(&s, &d) 377 | if err != nil { 378 | t.Fatal("Failed to delete destination:", err) 379 | } 380 | checkDestination(t, i, &s, &d, false) 381 | } 382 | } 383 | 384 | } 385 | } 386 | } 387 | 388 | func TestTimeouts(t *testing.T) { 389 | defer setupTestOSContext(t)() 390 | 391 | i, err := New("") 392 | if err != nil { 393 | t.Fatal("Failed to create IPVS handle:", err) 394 | } 395 | 396 | _, err = i.GetConfig() 397 | if err != nil { 398 | t.Fatal("Failed to get config:", err) 399 | } 400 | 401 | cfg := Config{66 * time.Second, 66 * time.Second, 66 * time.Second} 402 | err = i.SetConfig(&cfg) 403 | if err != nil { 404 | t.Fatal("Failed to set config:", err) 405 | } 406 | 407 | c2, err := i.GetConfig() 408 | if err != nil { 409 | t.Fatal("Failed to get config:", err) 410 | } 411 | if !reflect.DeepEqual(*c2, cfg) { 412 | t.Fatalf("expected: %+v, got: %+v", cfg, *c2) 413 | } 414 | 415 | // A timeout value 0 means that the current timeout value of the corresponding entry is preserved 416 | cfg = Config{77 * time.Second, 0 * time.Second, 77 * time.Second} 417 | err = i.SetConfig(&cfg) 418 | if err != nil { 419 | t.Fatal("Failed to set config:", err) 420 | } 421 | 422 | c3, err := i.GetConfig() 423 | if err != nil { 424 | t.Fatal("Failed to get config:", err) 425 | } 426 | expected := Config{77 * time.Second, 66 * time.Second, 77 * time.Second} 427 | if !reflect.DeepEqual(*c3, expected) { 428 | t.Fatalf("expected: %+v, got: %+v", expected, *c3) 429 | } 430 | } 431 | 432 | // setupTestOSContext joins a new network namespace, and returns its associated 433 | // teardown function. 434 | // 435 | // Example usage: 436 | // 437 | // defer setupTestOSContext(t)() 438 | func setupTestOSContext(t *testing.T) func() { 439 | t.Helper() 440 | runtime.LockOSThread() 441 | if err := syscall.Unshare(syscall.CLONE_NEWNET); err != nil { 442 | t.Fatalf("Failed to enter netns: %v", err) 443 | } 444 | 445 | fd, err := syscall.Open("/proc/self/ns/net", syscall.O_RDONLY, 0) 446 | if err != nil { 447 | t.Fatal("Failed to open netns file:", err) 448 | } 449 | 450 | // Since we are switching to a new test namespace make 451 | // sure to re-initialize initNs context 452 | ns.Init() 453 | 454 | runtime.LockOSThread() 455 | 456 | return func() { 457 | if err := syscall.Close(fd); err != nil { 458 | t.Logf("Warning: netns closing failed (%v)", err) 459 | } 460 | runtime.UnlockOSThread() 461 | } 462 | } 463 | -------------------------------------------------------------------------------- /internal/loadbalancer/loadbalancer_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.2.0 4 | // - protoc v3.21.12 5 | // source: internal/loadbalancer/loadbalancer.proto 6 | 7 | package loadbalancer 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | emptypb "google.golang.org/protobuf/types/known/emptypb" 15 | ) 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the grpc package it is being compiled against. 19 | // Requires gRPC-Go v1.32.0 or later. 20 | const _ = grpc.SupportPackageIsVersion7 21 | 22 | // LoadBalancerClient is the client API for LoadBalancer service. 23 | // 24 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 25 | type LoadBalancerClient interface { 26 | // Get all information about all defined Load Balancers 27 | GetLoadBalancers(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (LoadBalancer_GetLoadBalancersClient, error) 28 | // Get information about a specific Load Balancer. If no such name exists, return 29 | // an empty structure 30 | GetLoadBalancer(ctx context.Context, in *LoadBalancerName, opts ...grpc.CallOption) (*LoadBalancerInformation, error) 31 | Create(ctx context.Context, in *CreateLoadBalancer, opts ...grpc.CallOption) (*LoadBalancerInformation, error) 32 | Delete(ctx context.Context, in *LoadBalancerName, opts ...grpc.CallOption) (*Error, error) 33 | AddTarget(ctx context.Context, in *AddTargetRequest, opts ...grpc.CallOption) (*Error, error) 34 | DelTarget(ctx context.Context, in *DelTargetRequest, opts ...grpc.CallOption) (*Error, error) 35 | } 36 | 37 | type loadBalancerClient struct { 38 | cc grpc.ClientConnInterface 39 | } 40 | 41 | func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient { 42 | return &loadBalancerClient{cc} 43 | } 44 | 45 | func (c *loadBalancerClient) GetLoadBalancers(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (LoadBalancer_GetLoadBalancersClient, error) { 46 | stream, err := c.cc.NewStream(ctx, &LoadBalancer_ServiceDesc.Streams[0], "/loadbalancer.LoadBalancer/GetLoadBalancers", opts...) 47 | if err != nil { 48 | return nil, err 49 | } 50 | x := &loadBalancerGetLoadBalancersClient{stream} 51 | if err := x.ClientStream.SendMsg(in); err != nil { 52 | return nil, err 53 | } 54 | if err := x.ClientStream.CloseSend(); err != nil { 55 | return nil, err 56 | } 57 | return x, nil 58 | } 59 | 60 | type LoadBalancer_GetLoadBalancersClient interface { 61 | Recv() (*LoadBalancerInformation, error) 62 | grpc.ClientStream 63 | } 64 | 65 | type loadBalancerGetLoadBalancersClient struct { 66 | grpc.ClientStream 67 | } 68 | 69 | func (x *loadBalancerGetLoadBalancersClient) Recv() (*LoadBalancerInformation, error) { 70 | m := new(LoadBalancerInformation) 71 | if err := x.ClientStream.RecvMsg(m); err != nil { 72 | return nil, err 73 | } 74 | return m, nil 75 | } 76 | 77 | func (c *loadBalancerClient) GetLoadBalancer(ctx context.Context, in *LoadBalancerName, opts ...grpc.CallOption) (*LoadBalancerInformation, error) { 78 | out := new(LoadBalancerInformation) 79 | err := c.cc.Invoke(ctx, "/loadbalancer.LoadBalancer/GetLoadBalancer", in, out, opts...) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return out, nil 84 | } 85 | 86 | func (c *loadBalancerClient) Create(ctx context.Context, in *CreateLoadBalancer, opts ...grpc.CallOption) (*LoadBalancerInformation, error) { 87 | out := new(LoadBalancerInformation) 88 | err := c.cc.Invoke(ctx, "/loadbalancer.LoadBalancer/Create", in, out, opts...) 89 | if err != nil { 90 | return nil, err 91 | } 92 | return out, nil 93 | } 94 | 95 | func (c *loadBalancerClient) Delete(ctx context.Context, in *LoadBalancerName, opts ...grpc.CallOption) (*Error, error) { 96 | out := new(Error) 97 | err := c.cc.Invoke(ctx, "/loadbalancer.LoadBalancer/Delete", in, out, opts...) 98 | if err != nil { 99 | return nil, err 100 | } 101 | return out, nil 102 | } 103 | 104 | func (c *loadBalancerClient) AddTarget(ctx context.Context, in *AddTargetRequest, opts ...grpc.CallOption) (*Error, error) { 105 | out := new(Error) 106 | err := c.cc.Invoke(ctx, "/loadbalancer.LoadBalancer/AddTarget", in, out, opts...) 107 | if err != nil { 108 | return nil, err 109 | } 110 | return out, nil 111 | } 112 | 113 | func (c *loadBalancerClient) DelTarget(ctx context.Context, in *DelTargetRequest, opts ...grpc.CallOption) (*Error, error) { 114 | out := new(Error) 115 | err := c.cc.Invoke(ctx, "/loadbalancer.LoadBalancer/DelTarget", in, out, opts...) 116 | if err != nil { 117 | return nil, err 118 | } 119 | return out, nil 120 | } 121 | 122 | // LoadBalancerServer is the server API for LoadBalancer service. 123 | // All implementations must embed UnimplementedLoadBalancerServer 124 | // for forward compatibility 125 | type LoadBalancerServer interface { 126 | // Get all information about all defined Load Balancers 127 | GetLoadBalancers(*emptypb.Empty, LoadBalancer_GetLoadBalancersServer) error 128 | // Get information about a specific Load Balancer. If no such name exists, return 129 | // an empty structure 130 | GetLoadBalancer(context.Context, *LoadBalancerName) (*LoadBalancerInformation, error) 131 | Create(context.Context, *CreateLoadBalancer) (*LoadBalancerInformation, error) 132 | Delete(context.Context, *LoadBalancerName) (*Error, error) 133 | AddTarget(context.Context, *AddTargetRequest) (*Error, error) 134 | DelTarget(context.Context, *DelTargetRequest) (*Error, error) 135 | mustEmbedUnimplementedLoadBalancerServer() 136 | } 137 | 138 | // UnimplementedLoadBalancerServer must be embedded to have forward compatible implementations. 139 | type UnimplementedLoadBalancerServer struct { 140 | } 141 | 142 | func (UnimplementedLoadBalancerServer) GetLoadBalancers(*emptypb.Empty, LoadBalancer_GetLoadBalancersServer) error { 143 | return status.Errorf(codes.Unimplemented, "method GetLoadBalancers not implemented") 144 | } 145 | func (UnimplementedLoadBalancerServer) GetLoadBalancer(context.Context, *LoadBalancerName) (*LoadBalancerInformation, error) { 146 | return nil, status.Errorf(codes.Unimplemented, "method GetLoadBalancer not implemented") 147 | } 148 | func (UnimplementedLoadBalancerServer) Create(context.Context, *CreateLoadBalancer) (*LoadBalancerInformation, error) { 149 | return nil, status.Errorf(codes.Unimplemented, "method Create not implemented") 150 | } 151 | func (UnimplementedLoadBalancerServer) Delete(context.Context, *LoadBalancerName) (*Error, error) { 152 | return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") 153 | } 154 | func (UnimplementedLoadBalancerServer) AddTarget(context.Context, *AddTargetRequest) (*Error, error) { 155 | return nil, status.Errorf(codes.Unimplemented, "method AddTarget not implemented") 156 | } 157 | func (UnimplementedLoadBalancerServer) DelTarget(context.Context, *DelTargetRequest) (*Error, error) { 158 | return nil, status.Errorf(codes.Unimplemented, "method DelTarget not implemented") 159 | } 160 | func (UnimplementedLoadBalancerServer) mustEmbedUnimplementedLoadBalancerServer() {} 161 | 162 | // UnsafeLoadBalancerServer may be embedded to opt out of forward compatibility for this service. 163 | // Use of this interface is not recommended, as added methods to LoadBalancerServer will 164 | // result in compilation errors. 165 | type UnsafeLoadBalancerServer interface { 166 | mustEmbedUnimplementedLoadBalancerServer() 167 | } 168 | 169 | func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) { 170 | s.RegisterService(&LoadBalancer_ServiceDesc, srv) 171 | } 172 | 173 | func _LoadBalancer_GetLoadBalancers_Handler(srv interface{}, stream grpc.ServerStream) error { 174 | m := new(emptypb.Empty) 175 | if err := stream.RecvMsg(m); err != nil { 176 | return err 177 | } 178 | return srv.(LoadBalancerServer).GetLoadBalancers(m, &loadBalancerGetLoadBalancersServer{stream}) 179 | } 180 | 181 | type LoadBalancer_GetLoadBalancersServer interface { 182 | Send(*LoadBalancerInformation) error 183 | grpc.ServerStream 184 | } 185 | 186 | type loadBalancerGetLoadBalancersServer struct { 187 | grpc.ServerStream 188 | } 189 | 190 | func (x *loadBalancerGetLoadBalancersServer) Send(m *LoadBalancerInformation) error { 191 | return x.ServerStream.SendMsg(m) 192 | } 193 | 194 | func _LoadBalancer_GetLoadBalancer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 195 | in := new(LoadBalancerName) 196 | if err := dec(in); err != nil { 197 | return nil, err 198 | } 199 | if interceptor == nil { 200 | return srv.(LoadBalancerServer).GetLoadBalancer(ctx, in) 201 | } 202 | info := &grpc.UnaryServerInfo{ 203 | Server: srv, 204 | FullMethod: "/loadbalancer.LoadBalancer/GetLoadBalancer", 205 | } 206 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 207 | return srv.(LoadBalancerServer).GetLoadBalancer(ctx, req.(*LoadBalancerName)) 208 | } 209 | return interceptor(ctx, in, info, handler) 210 | } 211 | 212 | func _LoadBalancer_Create_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 213 | in := new(CreateLoadBalancer) 214 | if err := dec(in); err != nil { 215 | return nil, err 216 | } 217 | if interceptor == nil { 218 | return srv.(LoadBalancerServer).Create(ctx, in) 219 | } 220 | info := &grpc.UnaryServerInfo{ 221 | Server: srv, 222 | FullMethod: "/loadbalancer.LoadBalancer/Create", 223 | } 224 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 225 | return srv.(LoadBalancerServer).Create(ctx, req.(*CreateLoadBalancer)) 226 | } 227 | return interceptor(ctx, in, info, handler) 228 | } 229 | 230 | func _LoadBalancer_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 231 | in := new(LoadBalancerName) 232 | if err := dec(in); err != nil { 233 | return nil, err 234 | } 235 | if interceptor == nil { 236 | return srv.(LoadBalancerServer).Delete(ctx, in) 237 | } 238 | info := &grpc.UnaryServerInfo{ 239 | Server: srv, 240 | FullMethod: "/loadbalancer.LoadBalancer/Delete", 241 | } 242 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 243 | return srv.(LoadBalancerServer).Delete(ctx, req.(*LoadBalancerName)) 244 | } 245 | return interceptor(ctx, in, info, handler) 246 | } 247 | 248 | func _LoadBalancer_AddTarget_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 249 | in := new(AddTargetRequest) 250 | if err := dec(in); err != nil { 251 | return nil, err 252 | } 253 | if interceptor == nil { 254 | return srv.(LoadBalancerServer).AddTarget(ctx, in) 255 | } 256 | info := &grpc.UnaryServerInfo{ 257 | Server: srv, 258 | FullMethod: "/loadbalancer.LoadBalancer/AddTarget", 259 | } 260 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 261 | return srv.(LoadBalancerServer).AddTarget(ctx, req.(*AddTargetRequest)) 262 | } 263 | return interceptor(ctx, in, info, handler) 264 | } 265 | 266 | func _LoadBalancer_DelTarget_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 267 | in := new(DelTargetRequest) 268 | if err := dec(in); err != nil { 269 | return nil, err 270 | } 271 | if interceptor == nil { 272 | return srv.(LoadBalancerServer).DelTarget(ctx, in) 273 | } 274 | info := &grpc.UnaryServerInfo{ 275 | Server: srv, 276 | FullMethod: "/loadbalancer.LoadBalancer/DelTarget", 277 | } 278 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 279 | return srv.(LoadBalancerServer).DelTarget(ctx, req.(*DelTargetRequest)) 280 | } 281 | return interceptor(ctx, in, info, handler) 282 | } 283 | 284 | // LoadBalancer_ServiceDesc is the grpc.ServiceDesc for LoadBalancer service. 285 | // It's only intended for direct use with grpc.RegisterService, 286 | // and not to be introspected or modified (even as a copy) 287 | var LoadBalancer_ServiceDesc = grpc.ServiceDesc{ 288 | ServiceName: "loadbalancer.LoadBalancer", 289 | HandlerType: (*LoadBalancerServer)(nil), 290 | Methods: []grpc.MethodDesc{ 291 | { 292 | MethodName: "GetLoadBalancer", 293 | Handler: _LoadBalancer_GetLoadBalancer_Handler, 294 | }, 295 | { 296 | MethodName: "Create", 297 | Handler: _LoadBalancer_Create_Handler, 298 | }, 299 | { 300 | MethodName: "Delete", 301 | Handler: _LoadBalancer_Delete_Handler, 302 | }, 303 | { 304 | MethodName: "AddTarget", 305 | Handler: _LoadBalancer_AddTarget_Handler, 306 | }, 307 | { 308 | MethodName: "DelTarget", 309 | Handler: _LoadBalancer_DelTarget_Handler, 310 | }, 311 | }, 312 | Streams: []grpc.StreamDesc{ 313 | { 314 | StreamName: "GetLoadBalancers", 315 | Handler: _LoadBalancer_GetLoadBalancers_Handler, 316 | ServerStreams: true, 317 | }, 318 | }, 319 | Metadata: "internal/loadbalancer/loadbalancer.proto", 320 | } 321 | -------------------------------------------------------------------------------- /internal/loadbalancer/impl.go: -------------------------------------------------------------------------------- 1 | package loadbalancer 2 | 3 | import ( 4 | context "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net" 9 | "syscall" 10 | 11 | "github.com/knadh/koanf/v2" 12 | "github.com/rosedblabs/rosedb/v2" 13 | "github.com/vishvananda/netlink" 14 | "github.com/vishvananda/netns" 15 | emptypb "google.golang.org/protobuf/types/known/emptypb" 16 | 17 | "github.com/liorokman/proxmox-cloud-provider/internal/ipam" 18 | "github.com/liorokman/proxmox-cloud-provider/internal/ipvs" 19 | ) 20 | 21 | // Error codes that can appear in the Error.Code field 22 | const ( 23 | ErrSuccess = 0 24 | ErrNoSuchLB = (1 << 0) 25 | ErrNoSuchService = (1 << 1) 26 | ErrNoSuchIP = (1 << 2) 27 | ErrAddDestinationFailed = (1 << 3) 28 | ErrDelDestinationFailed = (1 << 4) 29 | ErrTargetAlreadyMapped = (1 << 5) 30 | ErrTargetNotMapped = (1 << 6) 31 | ErrIPAMError = (1 << 7) 32 | ) 33 | 34 | var ( 35 | errNoSuchLoadbalancer = fmt.Errorf("no such loadbalancer") 36 | ) 37 | 38 | type loadBalancerServer struct { 39 | UnimplementedLoadBalancerServer 40 | 41 | cidr *netlink.Addr 42 | db *rosedb.DB 43 | externalLink netlink.Link 44 | ns netns.NsHandle 45 | nlHandle *netlink.Handle 46 | ipvsHandle *ipvs.Handle 47 | ipam ipam.IPAM 48 | } 49 | 50 | type Intf interface { 51 | LoadBalancerServer 52 | Close() 53 | Restore() error 54 | } 55 | 56 | func (l *loadBalancerServer) Close() { 57 | l.ipvsHandle.Close() 58 | l.nlHandle.Delete() 59 | l.ns.Close() 60 | l.db.Close() 61 | } 62 | 63 | func NewServer(config *koanf.Koanf) (Intf, error) { 64 | 65 | cidr, err := netlink.ParseAddr(config.MustString("loadbalancer.ipam.cidr")) 66 | if err != nil { 67 | return nil, err 68 | } 69 | ns, err := netns.GetFromName(config.MustString("loadbalancer.namespace")) 70 | if err != nil { 71 | return nil, err 72 | } 73 | netlinkHandle, err := netlink.NewHandleAt(ns) 74 | if err != nil { 75 | ns.Close() 76 | return nil, err 77 | } 78 | netIf, err := netlinkHandle.LinkByName(config.MustString("loadbalancer.externalInterface")) 79 | if err != nil { 80 | netlinkHandle.Delete() 81 | ns.Close() 82 | return nil, err 83 | } 84 | ipvsHandle, err := ipvs.NewInNamespace(ns) 85 | if err != nil { 86 | netlinkHandle.Delete() 87 | ns.Close() 88 | return nil, err 89 | } 90 | options := rosedb.DefaultOptions 91 | options.DirPath = config.MustString("dbDir") 92 | // open a database 93 | db, err := rosedb.Open(options) 94 | if err != nil { 95 | ipvsHandle.Close() 96 | netlinkHandle.Delete() 97 | ns.Close() 98 | return nil, err 99 | } 100 | ipManagement, err := ipam.New(config, netlinkHandle, netIf) 101 | if err != nil { 102 | db.Close() 103 | ipvsHandle.Close() 104 | netlinkHandle.Delete() 105 | ns.Close() 106 | return nil, err 107 | } 108 | lbServer := &loadBalancerServer{ 109 | cidr: cidr, 110 | externalLink: netIf, 111 | db: db, 112 | ns: ns, 113 | nlHandle: netlinkHandle, 114 | ipvsHandle: ipvsHandle, 115 | ipam: ipManagement, 116 | } 117 | return lbServer, nil 118 | } 119 | 120 | func toIPVSService(l *LoadBalancerInformation, srcPort int32, protocol Protocol) *ipvs.Service { 121 | srcIP := net.ParseIP(l.IpAddr) 122 | var fam uint16 = syscall.AF_INET 123 | if srcIP.To4() == nil { 124 | fam = syscall.AF_INET6 125 | } 126 | var p uint16 = syscall.IPPROTO_TCP 127 | if protocol == Protocol_UDP { 128 | p = syscall.IPPROTO_UDP 129 | } 130 | return &ipvs.Service{ 131 | Address: srcIP, 132 | Port: uint16(srcPort), 133 | Protocol: p, 134 | AddressFamily: fam, 135 | SchedName: "rr", 136 | } 137 | } 138 | 139 | // Restore applies all the settings stored in the database to the current state 140 | func (l *loadBalancerServer) Restore() error { 141 | iterOptions := rosedb.DefaultIteratorOptions 142 | iter := l.db.NewIterator(iterOptions) 143 | defer iter.Close() 144 | for ; iter.Valid(); iter.Next() { 145 | val, err := iter.Value() 146 | if err != nil { 147 | return err 148 | } 149 | var currLB LoadBalancerInformation 150 | if err := json.Unmarshal(val, &currLB); err != nil { 151 | return err 152 | } 153 | if err := l.addAddrAlias(currLB.IpAddr); err != nil { 154 | return err 155 | } 156 | for port, targets := range currLB.Targets { 157 | if targets != nil { 158 | for _, target := range targets.Target { 159 | if _, err := l.mapTarget(&currLB, port, target); err != nil { 160 | return err 161 | } 162 | } 163 | } 164 | } 165 | } 166 | return nil 167 | } 168 | 169 | // Get all information about all defined Load Balancers 170 | func (l *loadBalancerServer) GetLoadBalancers(_ *emptypb.Empty, stream LoadBalancer_GetLoadBalancersServer) error { 171 | iterOptions := rosedb.DefaultIteratorOptions 172 | iter := l.db.NewIterator(iterOptions) 173 | defer iter.Close() 174 | for ; iter.Valid(); iter.Next() { 175 | val, err := iter.Value() 176 | if err != nil { 177 | return err 178 | } 179 | var curr LoadBalancerInformation 180 | if err := json.Unmarshal(val, &curr); err != nil { 181 | return err 182 | } 183 | if err := stream.Send(&curr); err != nil { 184 | return err 185 | } 186 | } 187 | return nil 188 | } 189 | 190 | // Get information about a specific Load Balancer 191 | func (l *loadBalancerServer) GetLoadBalancer(ctx context.Context, name *LoadBalancerName) (*LoadBalancerInformation, error) { 192 | 193 | curr, err := l.getLB(name.Name) 194 | if err != nil && err != errNoSuchLoadbalancer { 195 | return &LoadBalancerInformation{}, err 196 | } 197 | return curr, nil 198 | } 199 | 200 | func (l *loadBalancerServer) Create(ctx context.Context, clb *CreateLoadBalancer) (*LoadBalancerInformation, error) { 201 | 202 | log.Printf("Incoming request: %+v", clb) 203 | if _, err := l.db.Get([]byte(clb.Name)); err != nil && err != rosedb.ErrKeyNotFound { 204 | return nil, err 205 | } else if err == nil { 206 | return nil, fmt.Errorf("loadbalancer called %s already exists", clb.Name) 207 | } 208 | if clb.IpAddr == nil || *clb.IpAddr == "" { 209 | nextIP, err := l.ipam.Allocate() 210 | if err != nil { 211 | return nil, err 212 | } 213 | clb.IpAddr = asPtr(nextIP.String()) 214 | } 215 | 216 | lbInfo := &LoadBalancerInformation{ 217 | Name: clb.Name, 218 | IpAddr: *clb.IpAddr, 219 | Targets: map[int32]*TargetList{}, 220 | } 221 | 222 | // Create a new alias on the interface that matches the required IP 223 | // There could already be such an interface, since there might be another LB with the same IP and different port 224 | if err := l.addAddrAlias(lbInfo.IpAddr); err != nil { 225 | log.Printf("error adding an alias for the loadbalancer %s: %s", lbInfo.Name, err.Error()) 226 | return nil, err 227 | } 228 | 229 | lbInfoData, err := json.Marshal(lbInfo) 230 | if err != nil { 231 | return nil, err 232 | } 233 | err = l.db.Put([]byte(clb.Name), lbInfoData) 234 | return lbInfo, err 235 | } 236 | 237 | func (l *loadBalancerServer) addAddrAlias(addrString string) error { 238 | externalIP := net.ParseIP(addrString) 239 | if !l.ipam.Contains(externalIP) { 240 | return fmt.Errorf("Requested IP %s is not contained in the configured CIDR (%s)", addrString, l.cidr) 241 | } 242 | requiredAddr := &netlink.Addr{ 243 | IPNet: &net.IPNet{ 244 | IP: externalIP, 245 | Mask: l.cidr.Mask, 246 | }, 247 | } 248 | addrs, err := l.nlHandle.AddrList(l.externalLink, netlink.FAMILY_ALL) 249 | addrFound := false 250 | for _, addr := range addrs { 251 | addrFound = requiredAddr.Equal(addr) 252 | if addrFound { 253 | break 254 | } 255 | } 256 | if !addrFound { 257 | err = l.nlHandle.AddrAdd(l.externalLink, requiredAddr) 258 | if err != nil { 259 | return err 260 | } 261 | } 262 | return nil 263 | } 264 | 265 | func (l *loadBalancerServer) Delete(ctx context.Context, lbName *LoadBalancerName) (*Error, error) { 266 | lbInfo, err := l.getLB(lbName.Name) 267 | if err != nil { 268 | if err == errNoSuchLoadbalancer { 269 | return &Error{Message: fmt.Sprintf("no such loadbalancer %s", lbName.Name), Code: ErrNoSuchLB}, nil 270 | } 271 | return nil, err 272 | } 273 | 274 | retErr := &Error{} 275 | // 1. Remove the IPVS service 276 | for port, target := range lbInfo.Targets { 277 | protocols := map[Protocol]byte{} 278 | for _, v := range target.Target { 279 | protocols[v.Protocol] = 0 280 | } 281 | for p, _ := range protocols { 282 | srv := toIPVSService(lbInfo, port, p) 283 | if err := l.ipvsHandle.DelService(srv); err != nil { 284 | log.Printf("Error removing the ipvs service: %+v", err) 285 | retErr.Message = err.Error() 286 | retErr.Code = ErrNoSuchService 287 | } 288 | } 289 | } 290 | 291 | externalIP := net.ParseIP(lbInfo.IpAddr) 292 | // 2. remove the IP alias 293 | addrToDelete := &netlink.Addr{ 294 | IPNet: &net.IPNet{ 295 | IP: externalIP, 296 | Mask: l.cidr.Mask, 297 | }, 298 | } 299 | if err := l.nlHandle.AddrDel(l.externalLink, addrToDelete); err != nil { 300 | log.Printf("error removing the ip alias: %+v", err) 301 | retErr.Message = retErr.Message + "\n" + err.Error() 302 | retErr.Code |= ErrNoSuchIP 303 | } 304 | if err := l.ipam.Release(addrToDelete.IP); err != nil { 305 | log.Printf("error unallocating the ip from ipam: %+v", err) 306 | retErr.Message = retErr.Message + "\n" + err.Error() 307 | retErr.Code |= ErrIPAMError 308 | } 309 | 310 | if err := l.db.Delete([]byte(lbName.Name)); err != nil { 311 | return nil, err 312 | } 313 | 314 | return retErr, nil 315 | } 316 | 317 | func (l *loadBalancerServer) AddTarget(ctx context.Context, atr *AddTargetRequest) (*Error, error) { 318 | lbInfo, err := l.getLB(atr.LbName) 319 | if err != nil { 320 | if err == errNoSuchLoadbalancer { 321 | return &Error{Message: fmt.Sprintf("no such loadbalancer %s", atr.LbName), Code: ErrNoSuchLB}, nil 322 | } 323 | return nil, err 324 | } 325 | targetList, found := lbInfo.Targets[atr.SrcPort] 326 | if !found || targetList == nil { 327 | targetList = &TargetList{ 328 | Target: []*Target{}, 329 | } 330 | } 331 | for _, target := range targetList.Target { 332 | if target.Protocol == atr.Target.Protocol && 333 | target.DstIP == atr.Target.DstIP && 334 | target.DstPort == atr.Target.DstPort { 335 | 336 | return &Error{ 337 | Message: fmt.Sprintf("target %+v already mapped to load balancer %s", atr.Target, atr.LbName), 338 | Code: ErrTargetAlreadyMapped, 339 | }, nil 340 | } 341 | } 342 | 343 | ret, err := l.mapTarget(lbInfo, atr.SrcPort, atr.Target) 344 | if err != nil || ret != nil { 345 | return ret, err 346 | } 347 | 348 | targetList.Target = append(targetList.Target, atr.Target) 349 | lbInfo.Targets[atr.SrcPort] = targetList 350 | 351 | slbData, err := json.Marshal(lbInfo) 352 | if err != nil { 353 | return nil, err 354 | } 355 | if err := l.db.Put([]byte(atr.LbName), slbData); err != nil { 356 | return nil, err 357 | } 358 | return &Error{}, err 359 | } 360 | 361 | // mapTarget maps a given target to the provided loadbalancer on the provided 362 | // srcPort 363 | func (l *loadBalancerServer) mapTarget(lbInfo *LoadBalancerInformation, srcPort int32, target *Target) (*Error, error) { 364 | targetIP := net.ParseIP(target.DstIP) 365 | var fam uint16 = syscall.AF_INET 366 | if targetIP.To4() == nil { 367 | fam = syscall.AF_INET6 368 | } 369 | srv := toIPVSService(lbInfo, srcPort, target.Protocol) 370 | if !l.ipvsHandle.IsServicePresent(srv) { 371 | if err := l.ipvsHandle.NewService(srv); err != nil { 372 | return nil, err 373 | } 374 | } 375 | newDestination := &ipvs.Destination{ 376 | Address: targetIP, 377 | Port: uint16(target.DstPort), 378 | ConnectionFlags: ipvs.ConnFwdMasq, 379 | AddressFamily: fam, 380 | Weight: 1, 381 | } 382 | if err := l.ipvsHandle.NewDestination(srv, newDestination); err != nil { 383 | log.Printf("error adding a new destination %+v to service %+v: %+v", target, lbInfo, err) 384 | return &Error{ 385 | Message: err.Error(), 386 | Code: ErrAddDestinationFailed, 387 | }, nil 388 | } 389 | return nil, nil 390 | } 391 | 392 | func (l *loadBalancerServer) DelTarget(ctx context.Context, dtr *DelTargetRequest) (*Error, error) { 393 | lbInfo, err := l.getLB(dtr.LbName) 394 | if err != nil { 395 | if err == errNoSuchLoadbalancer { 396 | return &Error{Message: fmt.Sprintf("no such loadbalancer %s", dtr.LbName), Code: ErrNoSuchLB}, nil 397 | } 398 | return nil, err 399 | } 400 | targetList, found := lbInfo.Targets[dtr.SrcPort] 401 | if !found || targetList == nil { 402 | return &Error{ 403 | Message: fmt.Sprintf("target %+v not mapped to load balancer %s", dtr.Target, dtr.LbName), 404 | Code: ErrTargetNotMapped, 405 | }, nil 406 | } 407 | targetToRemove := -1 408 | for target, _ := range targetList.Target { 409 | if targetList.Target[target].Protocol == dtr.Target.Protocol && 410 | targetList.Target[target].DstIP == dtr.Target.DstIP && 411 | targetList.Target[target].DstPort == dtr.Target.DstPort { 412 | 413 | targetToRemove = target 414 | break 415 | } 416 | } 417 | if targetToRemove == -1 { 418 | return &Error{ 419 | Message: fmt.Sprintf("target %+v not mapped to load balancer %s on port %d", dtr.Target, dtr.LbName, dtr.SrcPort), 420 | Code: ErrTargetNotMapped, 421 | }, nil 422 | } 423 | 424 | target := targetList.Target[targetToRemove] 425 | 426 | targetIP := net.ParseIP(dtr.Target.DstIP) 427 | var fam uint16 = syscall.AF_INET 428 | if targetIP.To4() == nil { 429 | fam = syscall.AF_INET6 430 | } 431 | srv := toIPVSService(lbInfo, dtr.SrcPort, target.Protocol) 432 | dest := &ipvs.Destination{ 433 | Address: targetIP, 434 | Port: uint16(target.DstPort), 435 | ConnectionFlags: ipvs.ConnFwdMasq, 436 | AddressFamily: fam, 437 | Weight: 1, 438 | } 439 | if err := l.ipvsHandle.DelDestination(srv, dest); err != nil { 440 | log.Printf("error removing a destination %+v from service %+v: %+v", dtr.Target, lbInfo, err) 441 | return &Error{ 442 | Message: err.Error(), 443 | Code: ErrDelDestinationFailed, 444 | }, nil 445 | } 446 | targetList.Target = append(targetList.Target[:targetToRemove], targetList.Target[targetToRemove+1:]...) 447 | 448 | if len(targetList.Target) == 0 { 449 | // Delete the service 450 | if err := l.ipvsHandle.DelService(srv); err != nil { 451 | log.Printf("error removing a destination %+v from service %+v: %+v", dtr.Target, lbInfo, err) 452 | return &Error{ 453 | Message: err.Error(), 454 | Code: ErrDelDestinationFailed, 455 | }, nil 456 | } 457 | delete(lbInfo.Targets, dtr.SrcPort) 458 | } else { 459 | lbInfo.Targets[dtr.SrcPort] = targetList 460 | } 461 | 462 | slbData, err := json.Marshal(lbInfo) 463 | if err != nil { 464 | return nil, err 465 | } 466 | if err := l.db.Put([]byte(dtr.LbName), slbData); err != nil { 467 | return nil, err 468 | } 469 | return &Error{}, nil 470 | } 471 | 472 | func (l *loadBalancerServer) getLB(name string) (*LoadBalancerInformation, error) { 473 | lbinfoRaw, err := l.db.Get([]byte(name)) 474 | if err != nil { 475 | if err == rosedb.ErrKeyNotFound { 476 | return nil, errNoSuchLoadbalancer 477 | } 478 | return nil, err 479 | } 480 | var lbInfo LoadBalancerInformation 481 | if err := json.Unmarshal(lbinfoRaw, &lbInfo); err != nil { 482 | return nil, err 483 | } 484 | if lbInfo.Targets == nil { 485 | lbInfo.Targets = map[int32]*TargetList{} 486 | } 487 | return &lbInfo, nil 488 | } 489 | 490 | func asPtr[T any](s T) *T { 491 | return &s 492 | } 493 | -------------------------------------------------------------------------------- /internal/ipvs/netlink_linux.go: -------------------------------------------------------------------------------- 1 | package ipvs 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | "net" 9 | "os/exec" 10 | "strings" 11 | "sync" 12 | "sync/atomic" 13 | "syscall" 14 | "time" 15 | "unsafe" 16 | 17 | "github.com/sirupsen/logrus" 18 | "github.com/vishvananda/netlink/nl" 19 | "github.com/vishvananda/netns" 20 | ) 21 | 22 | // For Quick Reference IPVS related netlink message is described at the end of this file. 23 | var ( 24 | native = nl.NativeEndian() 25 | ipvsFamily int 26 | ipvsOnce sync.Once 27 | ) 28 | 29 | type genlMsgHdr struct { 30 | cmd uint8 31 | version uint8 32 | reserved uint16 33 | } 34 | 35 | type ipvsFlags struct { 36 | flags uint32 37 | mask uint32 38 | } 39 | 40 | func deserializeGenlMsg(b []byte) (hdr *genlMsgHdr) { 41 | return (*genlMsgHdr)(unsafe.Pointer(&b[0:unsafe.Sizeof(*hdr)][0])) 42 | } 43 | 44 | func (hdr *genlMsgHdr) Serialize() []byte { 45 | return (*(*[unsafe.Sizeof(*hdr)]byte)(unsafe.Pointer(hdr)))[:] 46 | } 47 | 48 | func (hdr *genlMsgHdr) Len() int { 49 | return int(unsafe.Sizeof(*hdr)) 50 | } 51 | 52 | func (f *ipvsFlags) Serialize() []byte { 53 | return (*(*[unsafe.Sizeof(*f)]byte)(unsafe.Pointer(f)))[:] 54 | } 55 | 56 | func (f *ipvsFlags) Len() int { 57 | return int(unsafe.Sizeof(*f)) 58 | } 59 | 60 | func setup() { 61 | ipvsOnce.Do(func() { 62 | var err error 63 | if out, err := exec.Command("modprobe", "-va", "ip_vs").CombinedOutput(); err != nil { 64 | logrus.Warnf("Running modprobe ip_vs failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) 65 | } 66 | 67 | ipvsFamily, err = getIPVSFamily() 68 | if err != nil { 69 | logrus.Error("Could not get ipvs family information from the kernel. It is possible that ipvs is not enabled in your kernel. Native loadbalancing will not work until this is fixed.") 70 | } 71 | }) 72 | } 73 | 74 | func fillService(s *Service) nl.NetlinkRequestData { 75 | cmdAttr := nl.NewRtAttr(ipvsCmdAttrService, nil) 76 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrAddressFamily, nl.Uint16Attr(s.AddressFamily)) 77 | if s.FWMark != 0 { 78 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrFWMark, nl.Uint32Attr(s.FWMark)) 79 | } else { 80 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrProtocol, nl.Uint16Attr(s.Protocol)) 81 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrAddress, rawIPData(s.Address)) 82 | 83 | // Port needs to be in network byte order. 84 | portBuf := new(bytes.Buffer) 85 | binary.Write(portBuf, binary.BigEndian, s.Port) 86 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrPort, portBuf.Bytes()) 87 | } 88 | 89 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrSchedName, nl.ZeroTerminated(s.SchedName)) 90 | if s.PEName != "" { 91 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrPEName, nl.ZeroTerminated(s.PEName)) 92 | } 93 | f := &ipvsFlags{ 94 | flags: s.Flags, 95 | mask: 0xFFFFFFFF, 96 | } 97 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrFlags, f.Serialize()) 98 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrTimeout, nl.Uint32Attr(s.Timeout)) 99 | nl.NewRtAttrChild(cmdAttr, ipvsSvcAttrNetmask, nl.Uint32Attr(s.Netmask)) 100 | return cmdAttr 101 | } 102 | 103 | func fillDestination(d *Destination) nl.NetlinkRequestData { 104 | cmdAttr := nl.NewRtAttr(ipvsCmdAttrDest, nil) 105 | 106 | nl.NewRtAttrChild(cmdAttr, ipvsDestAttrAddress, rawIPData(d.Address)) 107 | // Port needs to be in network byte order. 108 | portBuf := new(bytes.Buffer) 109 | binary.Write(portBuf, binary.BigEndian, d.Port) 110 | nl.NewRtAttrChild(cmdAttr, ipvsDestAttrPort, portBuf.Bytes()) 111 | 112 | nl.NewRtAttrChild(cmdAttr, ipvsDestAttrForwardingMethod, nl.Uint32Attr(d.ConnectionFlags&ConnectionFlagFwdMask)) 113 | nl.NewRtAttrChild(cmdAttr, ipvsDestAttrWeight, nl.Uint32Attr(uint32(d.Weight))) 114 | nl.NewRtAttrChild(cmdAttr, ipvsDestAttrUpperThreshold, nl.Uint32Attr(d.UpperThreshold)) 115 | nl.NewRtAttrChild(cmdAttr, ipvsDestAttrLowerThreshold, nl.Uint32Attr(d.LowerThreshold)) 116 | 117 | return cmdAttr 118 | } 119 | 120 | func (i *Handle) doCmdwithResponse(s *Service, d *Destination, cmd uint8) ([][]byte, error) { 121 | req := newIPVSRequest(cmd) 122 | req.Seq = atomic.AddUint32(&i.seq, 1) 123 | 124 | if s == nil { 125 | req.Flags |= syscall.NLM_F_DUMP // Flag to dump all messages 126 | req.AddData(nl.NewRtAttr(ipvsCmdAttrService, nil)) // Add a dummy attribute 127 | } else { 128 | req.AddData(fillService(s)) 129 | } 130 | 131 | if d == nil { 132 | if cmd == ipvsCmdGetDest { 133 | req.Flags |= syscall.NLM_F_DUMP 134 | } 135 | } else { 136 | req.AddData(fillDestination(d)) 137 | } 138 | 139 | res, err := execute(i.sock, req, 0) 140 | if err != nil { 141 | return [][]byte{}, err 142 | } 143 | 144 | return res, nil 145 | } 146 | 147 | func (i *Handle) doCmd(s *Service, d *Destination, cmd uint8) error { 148 | _, err := i.doCmdwithResponse(s, d, cmd) 149 | 150 | return err 151 | } 152 | 153 | func getIPVSFamily() (int, error) { 154 | sock, err := nl.GetNetlinkSocketAt(netns.None(), netns.None(), syscall.NETLINK_GENERIC) 155 | if err != nil { 156 | return 0, err 157 | } 158 | defer sock.Close() 159 | 160 | req := newGenlRequest(genlCtrlID, genlCtrlCmdGetFamily) 161 | req.AddData(nl.NewRtAttr(genlCtrlAttrFamilyName, nl.ZeroTerminated("IPVS"))) 162 | 163 | msgs, err := execute(sock, req, 0) 164 | if err != nil { 165 | return 0, err 166 | } 167 | 168 | for _, m := range msgs { 169 | hdr := deserializeGenlMsg(m) 170 | attrs, err := nl.ParseRouteAttr(m[hdr.Len():]) 171 | if err != nil { 172 | return 0, err 173 | } 174 | 175 | for _, attr := range attrs { 176 | switch int(attr.Attr.Type) { 177 | case genlCtrlAttrFamilyID: 178 | return int(native.Uint16(attr.Value[0:2])), nil 179 | } 180 | } 181 | } 182 | 183 | return 0, fmt.Errorf("no family id in the netlink response") 184 | } 185 | 186 | func rawIPData(ip net.IP) []byte { 187 | family := nl.GetIPFamily(ip) 188 | if family == nl.FAMILY_V4 { 189 | return ip.To4() 190 | } 191 | return ip 192 | } 193 | 194 | func newIPVSRequest(cmd uint8) *nl.NetlinkRequest { 195 | return newGenlRequest(ipvsFamily, cmd) 196 | } 197 | 198 | func newGenlRequest(familyID int, cmd uint8) *nl.NetlinkRequest { 199 | req := nl.NewNetlinkRequest(familyID, syscall.NLM_F_ACK) 200 | req.AddData(&genlMsgHdr{cmd: cmd, version: 1}) 201 | return req 202 | } 203 | 204 | func execute(s *nl.NetlinkSocket, req *nl.NetlinkRequest, resType uint16) ([][]byte, error) { 205 | if err := s.Send(req); err != nil { 206 | return nil, err 207 | } 208 | 209 | pid, err := s.GetPid() 210 | if err != nil { 211 | return nil, err 212 | } 213 | 214 | var res [][]byte 215 | 216 | done: 217 | for { 218 | msgs, _, err := s.Receive() 219 | if err != nil { 220 | if s.GetFd() == -1 { 221 | return nil, fmt.Errorf("Socket got closed on receive") 222 | } 223 | if err == syscall.EAGAIN { 224 | // timeout fired 225 | continue 226 | } 227 | return nil, err 228 | } 229 | for _, m := range msgs { 230 | if m.Header.Seq != req.Seq { 231 | continue 232 | } 233 | if m.Header.Pid != pid { 234 | return nil, fmt.Errorf("Wrong pid %d, expected %d", m.Header.Pid, pid) 235 | } 236 | if m.Header.Type == syscall.NLMSG_DONE { 237 | break done 238 | } 239 | if m.Header.Type == syscall.NLMSG_ERROR { 240 | error := int32(native.Uint32(m.Data[0:4])) 241 | if error == 0 { 242 | break done 243 | } 244 | return nil, syscall.Errno(-error) 245 | } 246 | if resType != 0 && m.Header.Type != resType { 247 | continue 248 | } 249 | res = append(res, m.Data) 250 | if m.Header.Flags&syscall.NLM_F_MULTI == 0 { 251 | break done 252 | } 253 | } 254 | } 255 | return res, nil 256 | } 257 | 258 | func parseIP(ip []byte, family uint16) (net.IP, error) { 259 | var resIP net.IP 260 | 261 | switch family { 262 | case syscall.AF_INET: 263 | resIP = (net.IP)(ip[:4]) 264 | case syscall.AF_INET6: 265 | resIP = (net.IP)(ip[:16]) 266 | default: 267 | return nil, fmt.Errorf("parseIP Error ip=%v", ip) 268 | 269 | } 270 | return resIP, nil 271 | } 272 | 273 | // parseStats 274 | func assembleStats(msg []byte) (SvcStats, error) { 275 | var s SvcStats 276 | 277 | attrs, err := nl.ParseRouteAttr(msg) 278 | if err != nil { 279 | return s, err 280 | } 281 | 282 | for _, attr := range attrs { 283 | attrType := int(attr.Attr.Type) 284 | switch attrType { 285 | case ipvsStatsConns: 286 | s.Connections = native.Uint32(attr.Value) 287 | case ipvsStatsPktsIn: 288 | s.PacketsIn = native.Uint32(attr.Value) 289 | case ipvsStatsPktsOut: 290 | s.PacketsOut = native.Uint32(attr.Value) 291 | case ipvsStatsBytesIn: 292 | s.BytesIn = native.Uint64(attr.Value) 293 | case ipvsStatsBytesOut: 294 | s.BytesOut = native.Uint64(attr.Value) 295 | case ipvsStatsCPS: 296 | s.CPS = native.Uint32(attr.Value) 297 | case ipvsStatsPPSIn: 298 | s.PPSIn = native.Uint32(attr.Value) 299 | case ipvsStatsPPSOut: 300 | s.PPSOut = native.Uint32(attr.Value) 301 | case ipvsStatsBPSIn: 302 | s.BPSIn = native.Uint32(attr.Value) 303 | case ipvsStatsBPSOut: 304 | s.BPSOut = native.Uint32(attr.Value) 305 | } 306 | } 307 | return s, nil 308 | } 309 | 310 | // assembleService assembles a services back from a hain of netlink attributes 311 | func assembleService(attrs []syscall.NetlinkRouteAttr) (*Service, error) { 312 | var s Service 313 | var addressBytes []byte 314 | 315 | for _, attr := range attrs { 316 | 317 | attrType := int(attr.Attr.Type) 318 | 319 | switch attrType { 320 | 321 | case ipvsSvcAttrAddressFamily: 322 | s.AddressFamily = native.Uint16(attr.Value) 323 | case ipvsSvcAttrProtocol: 324 | s.Protocol = native.Uint16(attr.Value) 325 | case ipvsSvcAttrAddress: 326 | addressBytes = attr.Value 327 | case ipvsSvcAttrPort: 328 | s.Port = binary.BigEndian.Uint16(attr.Value) 329 | case ipvsSvcAttrFWMark: 330 | s.FWMark = native.Uint32(attr.Value) 331 | case ipvsSvcAttrSchedName: 332 | s.SchedName = nl.BytesToString(attr.Value) 333 | case ipvsSvcAttrFlags: 334 | s.Flags = native.Uint32(attr.Value) 335 | case ipvsSvcAttrTimeout: 336 | s.Timeout = native.Uint32(attr.Value) 337 | case ipvsSvcAttrNetmask: 338 | s.Netmask = native.Uint32(attr.Value) 339 | case ipvsSvcAttrStats: 340 | stats, err := assembleStats(attr.Value) 341 | if err != nil { 342 | return nil, err 343 | } 344 | s.Stats = stats 345 | } 346 | 347 | } 348 | 349 | // parse Address after parse AddressFamily incase of parseIP error 350 | if addressBytes != nil { 351 | ip, err := parseIP(addressBytes, s.AddressFamily) 352 | if err != nil { 353 | return nil, err 354 | } 355 | s.Address = ip 356 | } 357 | 358 | return &s, nil 359 | } 360 | 361 | // parseService given a ipvs netlink response this function will respond with a valid service entry, an error otherwise 362 | func (i *Handle) parseService(msg []byte) (*Service, error) { 363 | var s *Service 364 | 365 | // Remove General header for this message and parse the NetLink message 366 | hdr := deserializeGenlMsg(msg) 367 | NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) 368 | if err != nil { 369 | return nil, err 370 | } 371 | if len(NetLinkAttrs) == 0 { 372 | return nil, fmt.Errorf("error no valid netlink message found while parsing service record") 373 | } 374 | 375 | // Now Parse and get IPVS related attributes messages packed in this message. 376 | ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) 377 | if err != nil { 378 | return nil, err 379 | } 380 | 381 | // Assemble all the IPVS related attribute messages and create a service record 382 | s, err = assembleService(ipvsAttrs) 383 | if err != nil { 384 | return nil, err 385 | } 386 | 387 | return s, nil 388 | } 389 | 390 | // doGetServicesCmd a wrapper which could be used commonly for both GetServices() and GetService(*Service) 391 | func (i *Handle) doGetServicesCmd(svc *Service) ([]*Service, error) { 392 | var res []*Service 393 | 394 | msgs, err := i.doCmdwithResponse(svc, nil, ipvsCmdGetService) 395 | if err != nil { 396 | return nil, err 397 | } 398 | 399 | for _, msg := range msgs { 400 | srv, err := i.parseService(msg) 401 | if err != nil { 402 | return nil, err 403 | } 404 | res = append(res, srv) 405 | } 406 | 407 | return res, nil 408 | } 409 | 410 | // doCmdWithoutAttr a simple wrapper of netlink socket execute command 411 | func (i *Handle) doCmdWithoutAttr(cmd uint8) ([][]byte, error) { 412 | req := newIPVSRequest(cmd) 413 | req.Seq = atomic.AddUint32(&i.seq, 1) 414 | return execute(i.sock, req, 0) 415 | } 416 | 417 | func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error) { 418 | var d Destination 419 | var addressBytes []byte 420 | 421 | for _, attr := range attrs { 422 | 423 | attrType := int(attr.Attr.Type) 424 | 425 | switch attrType { 426 | 427 | case ipvsDestAttrAddressFamily: 428 | d.AddressFamily = native.Uint16(attr.Value) 429 | case ipvsDestAttrAddress: 430 | addressBytes = attr.Value 431 | case ipvsDestAttrPort: 432 | d.Port = binary.BigEndian.Uint16(attr.Value) 433 | case ipvsDestAttrForwardingMethod: 434 | d.ConnectionFlags = native.Uint32(attr.Value) 435 | case ipvsDestAttrWeight: 436 | d.Weight = int(native.Uint16(attr.Value)) 437 | case ipvsDestAttrUpperThreshold: 438 | d.UpperThreshold = native.Uint32(attr.Value) 439 | case ipvsDestAttrLowerThreshold: 440 | d.LowerThreshold = native.Uint32(attr.Value) 441 | case ipvsDestAttrActiveConnections: 442 | d.ActiveConnections = int(native.Uint32(attr.Value)) 443 | case ipvsDestAttrInactiveConnections: 444 | d.InactiveConnections = int(native.Uint32(attr.Value)) 445 | case ipvsDestAttrStats: 446 | stats, err := assembleStats(attr.Value) 447 | if err != nil { 448 | return nil, err 449 | } 450 | d.Stats = DstStats(stats) 451 | } 452 | } 453 | 454 | // in older kernels (< 3.18), the destination address family attribute doesn't exist so we must 455 | // assume it based on the destination address provided. 456 | if d.AddressFamily == 0 { 457 | // we can't check the address family using net stdlib because netlink returns 458 | // IPv4 addresses as the first 4 bytes in a []byte of length 16 where as 459 | // stdlib expects it as the last 4 bytes. 460 | addressFamily, err := getIPFamily(addressBytes) 461 | if err != nil { 462 | return nil, err 463 | } 464 | d.AddressFamily = addressFamily 465 | } 466 | 467 | // parse Address after parse AddressFamily incase of parseIP error 468 | if addressBytes != nil { 469 | ip, err := parseIP(addressBytes, d.AddressFamily) 470 | if err != nil { 471 | return nil, err 472 | } 473 | d.Address = ip 474 | } 475 | 476 | return &d, nil 477 | } 478 | 479 | // getIPFamily parses the IP family based on raw data from netlink. 480 | // For AF_INET, netlink will set the first 4 bytes with trailing zeros 481 | // 482 | // 10.0.0.1 -> [10 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0] 483 | // 484 | // For AF_INET6, the full 16 byte array is used: 485 | // 486 | // 2001:db8:3c4d:15::1a00 -> [32 1 13 184 60 77 0 21 0 0 0 0 0 0 26 0] 487 | func getIPFamily(address []byte) (uint16, error) { 488 | if len(address) == 4 { 489 | return syscall.AF_INET, nil 490 | } 491 | 492 | if isZeros(address) { 493 | return 0, errors.New("could not parse IP family from address data") 494 | } 495 | 496 | // assume IPv4 if first 4 bytes are non-zero but rest of the data is trailing zeros 497 | if !isZeros(address[:4]) && isZeros(address[4:]) { 498 | return syscall.AF_INET, nil 499 | } 500 | 501 | return syscall.AF_INET6, nil 502 | } 503 | 504 | func isZeros(b []byte) bool { 505 | for i := 0; i < len(b); i++ { 506 | if b[i] != 0 { 507 | return false 508 | } 509 | } 510 | return true 511 | } 512 | 513 | // parseDestination given a ipvs netlink response this function will respond with a valid destination entry, an error otherwise 514 | func (i *Handle) parseDestination(msg []byte) (*Destination, error) { 515 | var dst *Destination 516 | 517 | // Remove General header for this message 518 | hdr := deserializeGenlMsg(msg) 519 | NetLinkAttrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) 520 | if err != nil { 521 | return nil, err 522 | } 523 | if len(NetLinkAttrs) == 0 { 524 | return nil, fmt.Errorf("error no valid netlink message found while parsing destination record") 525 | } 526 | 527 | // Now Parse and get IPVS related attributes messages packed in this message. 528 | ipvsAttrs, err := nl.ParseRouteAttr(NetLinkAttrs[0].Value) 529 | if err != nil { 530 | return nil, err 531 | } 532 | 533 | // Assemble netlink attributes and create a Destination record 534 | dst, err = assembleDestination(ipvsAttrs) 535 | if err != nil { 536 | return nil, err 537 | } 538 | 539 | return dst, nil 540 | } 541 | 542 | // doGetDestinationsCmd a wrapper function to be used by GetDestinations and GetDestination(d) apis 543 | func (i *Handle) doGetDestinationsCmd(s *Service, d *Destination) ([]*Destination, error) { 544 | var res []*Destination 545 | 546 | msgs, err := i.doCmdwithResponse(s, d, ipvsCmdGetDest) 547 | if err != nil { 548 | return nil, err 549 | } 550 | 551 | for _, msg := range msgs { 552 | dest, err := i.parseDestination(msg) 553 | if err != nil { 554 | return res, err 555 | } 556 | res = append(res, dest) 557 | } 558 | return res, nil 559 | } 560 | 561 | // parseConfig given a ipvs netlink response this function will respond with a valid config entry, an error otherwise 562 | func (i *Handle) parseConfig(msg []byte) (*Config, error) { 563 | var c Config 564 | 565 | // Remove General header for this message 566 | hdr := deserializeGenlMsg(msg) 567 | attrs, err := nl.ParseRouteAttr(msg[hdr.Len():]) 568 | if err != nil { 569 | return nil, err 570 | } 571 | 572 | for _, attr := range attrs { 573 | attrType := int(attr.Attr.Type) 574 | switch attrType { 575 | case ipvsCmdAttrTimeoutTCP: 576 | c.TimeoutTCP = time.Duration(native.Uint32(attr.Value)) * time.Second 577 | case ipvsCmdAttrTimeoutTCPFin: 578 | c.TimeoutTCPFin = time.Duration(native.Uint32(attr.Value)) * time.Second 579 | case ipvsCmdAttrTimeoutUDP: 580 | c.TimeoutUDP = time.Duration(native.Uint32(attr.Value)) * time.Second 581 | } 582 | } 583 | 584 | return &c, nil 585 | } 586 | 587 | // doGetConfigCmd a wrapper function to be used by GetConfig 588 | func (i *Handle) doGetConfigCmd() (*Config, error) { 589 | msg, err := i.doCmdWithoutAttr(ipvsCmdGetConfig) 590 | if err != nil { 591 | return nil, err 592 | } 593 | 594 | res, err := i.parseConfig(msg[0]) 595 | if err != nil { 596 | return res, err 597 | } 598 | return res, nil 599 | } 600 | 601 | // doSetConfigCmd a wrapper function to be used by SetConfig 602 | func (i *Handle) doSetConfigCmd(c *Config) error { 603 | req := newIPVSRequest(ipvsCmdSetConfig) 604 | req.Seq = atomic.AddUint32(&i.seq, 1) 605 | 606 | req.AddData(nl.NewRtAttr(ipvsCmdAttrTimeoutTCP, nl.Uint32Attr(uint32(c.TimeoutTCP.Seconds())))) 607 | req.AddData(nl.NewRtAttr(ipvsCmdAttrTimeoutTCPFin, nl.Uint32Attr(uint32(c.TimeoutTCPFin.Seconds())))) 608 | req.AddData(nl.NewRtAttr(ipvsCmdAttrTimeoutUDP, nl.Uint32Attr(uint32(c.TimeoutUDP.Seconds())))) 609 | 610 | _, err := execute(i.sock, req, 0) 611 | 612 | return err 613 | } 614 | 615 | // IPVS related netlink message format explained 616 | 617 | /* EACH NETLINK MSG is of the below format, this is what we will receive from execute() api. 618 | If we have multiple netlink objects to process like GetServices() etc., execute() will 619 | supply an array of this below object 620 | 621 | NETLINK MSG 622 | |-----------------------------------| 623 | 0 1 2 3 624 | |--------|--------|--------|--------| - 625 | | CMD ID | VER | RESERVED | |==> General Message Header represented by genlMsgHdr 626 | |-----------------------------------| - 627 | | ATTR LEN | ATTR TYPE | | 628 | |-----------------------------------| | 629 | | | | 630 | | VALUE | | 631 | | []byte Array of IPVS MSG | |==> Attribute Message represented by syscall.NetlinkRouteAttr 632 | | PADDED BY 4 BYTES | | 633 | | | | 634 | |-----------------------------------| - 635 | 636 | 637 | Once We strip genlMsgHdr from above NETLINK MSG, we should parse the VALUE. 638 | VALUE will have an array of netlink attributes (syscall.NetlinkRouteAttr) such that each attribute will 639 | represent a "Service" or "Destination" object's field. If we assemble these attributes we can construct 640 | Service or Destination. 641 | 642 | IPVS MSG 643 | |-----------------------------------| 644 | 0 1 2 3 645 | |--------|--------|--------|--------| 646 | | ATTR LEN | ATTR TYPE | 647 | |-----------------------------------| 648 | | | 649 | | | 650 | | []byte IPVS ATTRIBUTE BY 4 BYTES | 651 | | | 652 | | | 653 | |-----------------------------------| 654 | NEXT ATTRIBUTE 655 | |-----------------------------------| 656 | | ATTR LEN | ATTR TYPE | 657 | |-----------------------------------| 658 | | | 659 | | | 660 | | []byte IPVS ATTRIBUTE BY 4 BYTES | 661 | | | 662 | | | 663 | |-----------------------------------| 664 | NEXT ATTRIBUTE 665 | |-----------------------------------| 666 | | ATTR LEN | ATTR TYPE | 667 | |-----------------------------------| 668 | | | 669 | | | 670 | | []byte IPVS ATTRIBUTE BY 4 BYTES | 671 | | | 672 | | | 673 | |-----------------------------------| 674 | 675 | */ 676 | -------------------------------------------------------------------------------- /internal/loadbalancer/loadbalancer.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.28.1 4 | // protoc v3.21.12 5 | // source: internal/loadbalancer/loadbalancer.proto 6 | 7 | package loadbalancer 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | emptypb "google.golang.org/protobuf/types/known/emptypb" 13 | reflect "reflect" 14 | sync "sync" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | type Protocol int32 25 | 26 | const ( 27 | Protocol_TCP Protocol = 0 28 | Protocol_UDP Protocol = 1 29 | ) 30 | 31 | // Enum value maps for Protocol. 32 | var ( 33 | Protocol_name = map[int32]string{ 34 | 0: "TCP", 35 | 1: "UDP", 36 | } 37 | Protocol_value = map[string]int32{ 38 | "TCP": 0, 39 | "UDP": 1, 40 | } 41 | ) 42 | 43 | func (x Protocol) Enum() *Protocol { 44 | p := new(Protocol) 45 | *p = x 46 | return p 47 | } 48 | 49 | func (x Protocol) String() string { 50 | return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 51 | } 52 | 53 | func (Protocol) Descriptor() protoreflect.EnumDescriptor { 54 | return file_internal_loadbalancer_loadbalancer_proto_enumTypes[0].Descriptor() 55 | } 56 | 57 | func (Protocol) Type() protoreflect.EnumType { 58 | return &file_internal_loadbalancer_loadbalancer_proto_enumTypes[0] 59 | } 60 | 61 | func (x Protocol) Number() protoreflect.EnumNumber { 62 | return protoreflect.EnumNumber(x) 63 | } 64 | 65 | // Deprecated: Use Protocol.Descriptor instead. 66 | func (Protocol) EnumDescriptor() ([]byte, []int) { 67 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{0} 68 | } 69 | 70 | type Error struct { 71 | state protoimpl.MessageState 72 | sizeCache protoimpl.SizeCache 73 | unknownFields protoimpl.UnknownFields 74 | 75 | Code uint32 `protobuf:"varint,1,opt,name=Code,proto3" json:"Code,omitempty"` 76 | Message string `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` 77 | } 78 | 79 | func (x *Error) Reset() { 80 | *x = Error{} 81 | if protoimpl.UnsafeEnabled { 82 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[0] 83 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 84 | ms.StoreMessageInfo(mi) 85 | } 86 | } 87 | 88 | func (x *Error) String() string { 89 | return protoimpl.X.MessageStringOf(x) 90 | } 91 | 92 | func (*Error) ProtoMessage() {} 93 | 94 | func (x *Error) ProtoReflect() protoreflect.Message { 95 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[0] 96 | if protoimpl.UnsafeEnabled && x != nil { 97 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 98 | if ms.LoadMessageInfo() == nil { 99 | ms.StoreMessageInfo(mi) 100 | } 101 | return ms 102 | } 103 | return mi.MessageOf(x) 104 | } 105 | 106 | // Deprecated: Use Error.ProtoReflect.Descriptor instead. 107 | func (*Error) Descriptor() ([]byte, []int) { 108 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{0} 109 | } 110 | 111 | func (x *Error) GetCode() uint32 { 112 | if x != nil { 113 | return x.Code 114 | } 115 | return 0 116 | } 117 | 118 | func (x *Error) GetMessage() string { 119 | if x != nil { 120 | return x.Message 121 | } 122 | return "" 123 | } 124 | 125 | type LoadBalancerName struct { 126 | state protoimpl.MessageState 127 | sizeCache protoimpl.SizeCache 128 | unknownFields protoimpl.UnknownFields 129 | 130 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` 131 | } 132 | 133 | func (x *LoadBalancerName) Reset() { 134 | *x = LoadBalancerName{} 135 | if protoimpl.UnsafeEnabled { 136 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[1] 137 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 138 | ms.StoreMessageInfo(mi) 139 | } 140 | } 141 | 142 | func (x *LoadBalancerName) String() string { 143 | return protoimpl.X.MessageStringOf(x) 144 | } 145 | 146 | func (*LoadBalancerName) ProtoMessage() {} 147 | 148 | func (x *LoadBalancerName) ProtoReflect() protoreflect.Message { 149 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[1] 150 | if protoimpl.UnsafeEnabled && x != nil { 151 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 152 | if ms.LoadMessageInfo() == nil { 153 | ms.StoreMessageInfo(mi) 154 | } 155 | return ms 156 | } 157 | return mi.MessageOf(x) 158 | } 159 | 160 | // Deprecated: Use LoadBalancerName.ProtoReflect.Descriptor instead. 161 | func (*LoadBalancerName) Descriptor() ([]byte, []int) { 162 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{1} 163 | } 164 | 165 | func (x *LoadBalancerName) GetName() string { 166 | if x != nil { 167 | return x.Name 168 | } 169 | return "" 170 | } 171 | 172 | type LoadBalancerInformation struct { 173 | state protoimpl.MessageState 174 | sizeCache protoimpl.SizeCache 175 | unknownFields protoimpl.UnknownFields 176 | 177 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` 178 | IpAddr string `protobuf:"bytes,2,opt,name=ip_addr,json=ipAddr,proto3" json:"ip_addr,omitempty"` 179 | Targets map[int32]*TargetList `protobuf:"bytes,3,rep,name=targets,proto3" json:"targets,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` 180 | } 181 | 182 | func (x *LoadBalancerInformation) Reset() { 183 | *x = LoadBalancerInformation{} 184 | if protoimpl.UnsafeEnabled { 185 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[2] 186 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 187 | ms.StoreMessageInfo(mi) 188 | } 189 | } 190 | 191 | func (x *LoadBalancerInformation) String() string { 192 | return protoimpl.X.MessageStringOf(x) 193 | } 194 | 195 | func (*LoadBalancerInformation) ProtoMessage() {} 196 | 197 | func (x *LoadBalancerInformation) ProtoReflect() protoreflect.Message { 198 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[2] 199 | if protoimpl.UnsafeEnabled && x != nil { 200 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 201 | if ms.LoadMessageInfo() == nil { 202 | ms.StoreMessageInfo(mi) 203 | } 204 | return ms 205 | } 206 | return mi.MessageOf(x) 207 | } 208 | 209 | // Deprecated: Use LoadBalancerInformation.ProtoReflect.Descriptor instead. 210 | func (*LoadBalancerInformation) Descriptor() ([]byte, []int) { 211 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{2} 212 | } 213 | 214 | func (x *LoadBalancerInformation) GetName() string { 215 | if x != nil { 216 | return x.Name 217 | } 218 | return "" 219 | } 220 | 221 | func (x *LoadBalancerInformation) GetIpAddr() string { 222 | if x != nil { 223 | return x.IpAddr 224 | } 225 | return "" 226 | } 227 | 228 | func (x *LoadBalancerInformation) GetTargets() map[int32]*TargetList { 229 | if x != nil { 230 | return x.Targets 231 | } 232 | return nil 233 | } 234 | 235 | type CreateLoadBalancer struct { 236 | state protoimpl.MessageState 237 | sizeCache protoimpl.SizeCache 238 | unknownFields protoimpl.UnknownFields 239 | 240 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` 241 | // If an ip_addr is requested, try to use it. Otherwise, an unused IP will be allocated. 242 | IpAddr *string `protobuf:"bytes,2,opt,name=ip_addr,json=ipAddr,proto3,oneof" json:"ip_addr,omitempty"` 243 | } 244 | 245 | func (x *CreateLoadBalancer) Reset() { 246 | *x = CreateLoadBalancer{} 247 | if protoimpl.UnsafeEnabled { 248 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[3] 249 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 250 | ms.StoreMessageInfo(mi) 251 | } 252 | } 253 | 254 | func (x *CreateLoadBalancer) String() string { 255 | return protoimpl.X.MessageStringOf(x) 256 | } 257 | 258 | func (*CreateLoadBalancer) ProtoMessage() {} 259 | 260 | func (x *CreateLoadBalancer) ProtoReflect() protoreflect.Message { 261 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[3] 262 | if protoimpl.UnsafeEnabled && x != nil { 263 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 264 | if ms.LoadMessageInfo() == nil { 265 | ms.StoreMessageInfo(mi) 266 | } 267 | return ms 268 | } 269 | return mi.MessageOf(x) 270 | } 271 | 272 | // Deprecated: Use CreateLoadBalancer.ProtoReflect.Descriptor instead. 273 | func (*CreateLoadBalancer) Descriptor() ([]byte, []int) { 274 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{3} 275 | } 276 | 277 | func (x *CreateLoadBalancer) GetName() string { 278 | if x != nil { 279 | return x.Name 280 | } 281 | return "" 282 | } 283 | 284 | func (x *CreateLoadBalancer) GetIpAddr() string { 285 | if x != nil && x.IpAddr != nil { 286 | return *x.IpAddr 287 | } 288 | return "" 289 | } 290 | 291 | type DelTargetRequest struct { 292 | state protoimpl.MessageState 293 | sizeCache protoimpl.SizeCache 294 | unknownFields protoimpl.UnknownFields 295 | 296 | LbName string `protobuf:"bytes,1,opt,name=lb_name,json=lbName,proto3" json:"lb_name,omitempty"` 297 | SrcPort int32 `protobuf:"varint,2,opt,name=srcPort,proto3" json:"srcPort,omitempty"` 298 | Target *Target `protobuf:"bytes,3,opt,name=target,proto3" json:"target,omitempty"` 299 | } 300 | 301 | func (x *DelTargetRequest) Reset() { 302 | *x = DelTargetRequest{} 303 | if protoimpl.UnsafeEnabled { 304 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[4] 305 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 306 | ms.StoreMessageInfo(mi) 307 | } 308 | } 309 | 310 | func (x *DelTargetRequest) String() string { 311 | return protoimpl.X.MessageStringOf(x) 312 | } 313 | 314 | func (*DelTargetRequest) ProtoMessage() {} 315 | 316 | func (x *DelTargetRequest) ProtoReflect() protoreflect.Message { 317 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[4] 318 | if protoimpl.UnsafeEnabled && x != nil { 319 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 320 | if ms.LoadMessageInfo() == nil { 321 | ms.StoreMessageInfo(mi) 322 | } 323 | return ms 324 | } 325 | return mi.MessageOf(x) 326 | } 327 | 328 | // Deprecated: Use DelTargetRequest.ProtoReflect.Descriptor instead. 329 | func (*DelTargetRequest) Descriptor() ([]byte, []int) { 330 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{4} 331 | } 332 | 333 | func (x *DelTargetRequest) GetLbName() string { 334 | if x != nil { 335 | return x.LbName 336 | } 337 | return "" 338 | } 339 | 340 | func (x *DelTargetRequest) GetSrcPort() int32 { 341 | if x != nil { 342 | return x.SrcPort 343 | } 344 | return 0 345 | } 346 | 347 | func (x *DelTargetRequest) GetTarget() *Target { 348 | if x != nil { 349 | return x.Target 350 | } 351 | return nil 352 | } 353 | 354 | type AddTargetRequest struct { 355 | state protoimpl.MessageState 356 | sizeCache protoimpl.SizeCache 357 | unknownFields protoimpl.UnknownFields 358 | 359 | LbName string `protobuf:"bytes,1,opt,name=lb_name,json=lbName,proto3" json:"lb_name,omitempty"` 360 | SrcPort int32 `protobuf:"varint,2,opt,name=srcPort,proto3" json:"srcPort,omitempty"` 361 | Target *Target `protobuf:"bytes,3,opt,name=target,proto3" json:"target,omitempty"` 362 | } 363 | 364 | func (x *AddTargetRequest) Reset() { 365 | *x = AddTargetRequest{} 366 | if protoimpl.UnsafeEnabled { 367 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[5] 368 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 369 | ms.StoreMessageInfo(mi) 370 | } 371 | } 372 | 373 | func (x *AddTargetRequest) String() string { 374 | return protoimpl.X.MessageStringOf(x) 375 | } 376 | 377 | func (*AddTargetRequest) ProtoMessage() {} 378 | 379 | func (x *AddTargetRequest) ProtoReflect() protoreflect.Message { 380 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[5] 381 | if protoimpl.UnsafeEnabled && x != nil { 382 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 383 | if ms.LoadMessageInfo() == nil { 384 | ms.StoreMessageInfo(mi) 385 | } 386 | return ms 387 | } 388 | return mi.MessageOf(x) 389 | } 390 | 391 | // Deprecated: Use AddTargetRequest.ProtoReflect.Descriptor instead. 392 | func (*AddTargetRequest) Descriptor() ([]byte, []int) { 393 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{5} 394 | } 395 | 396 | func (x *AddTargetRequest) GetLbName() string { 397 | if x != nil { 398 | return x.LbName 399 | } 400 | return "" 401 | } 402 | 403 | func (x *AddTargetRequest) GetSrcPort() int32 { 404 | if x != nil { 405 | return x.SrcPort 406 | } 407 | return 0 408 | } 409 | 410 | func (x *AddTargetRequest) GetTarget() *Target { 411 | if x != nil { 412 | return x.Target 413 | } 414 | return nil 415 | } 416 | 417 | type Target struct { 418 | state protoimpl.MessageState 419 | sizeCache protoimpl.SizeCache 420 | unknownFields protoimpl.UnknownFields 421 | 422 | Protocol Protocol `protobuf:"varint,1,opt,name=protocol,proto3,enum=loadbalancer.Protocol" json:"protocol,omitempty"` 423 | DstIP string `protobuf:"bytes,2,opt,name=dstIP,proto3" json:"dstIP,omitempty"` 424 | DstPort int32 `protobuf:"varint,3,opt,name=dstPort,proto3" json:"dstPort,omitempty"` 425 | } 426 | 427 | func (x *Target) Reset() { 428 | *x = Target{} 429 | if protoimpl.UnsafeEnabled { 430 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[6] 431 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 432 | ms.StoreMessageInfo(mi) 433 | } 434 | } 435 | 436 | func (x *Target) String() string { 437 | return protoimpl.X.MessageStringOf(x) 438 | } 439 | 440 | func (*Target) ProtoMessage() {} 441 | 442 | func (x *Target) ProtoReflect() protoreflect.Message { 443 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[6] 444 | if protoimpl.UnsafeEnabled && x != nil { 445 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 446 | if ms.LoadMessageInfo() == nil { 447 | ms.StoreMessageInfo(mi) 448 | } 449 | return ms 450 | } 451 | return mi.MessageOf(x) 452 | } 453 | 454 | // Deprecated: Use Target.ProtoReflect.Descriptor instead. 455 | func (*Target) Descriptor() ([]byte, []int) { 456 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{6} 457 | } 458 | 459 | func (x *Target) GetProtocol() Protocol { 460 | if x != nil { 461 | return x.Protocol 462 | } 463 | return Protocol_TCP 464 | } 465 | 466 | func (x *Target) GetDstIP() string { 467 | if x != nil { 468 | return x.DstIP 469 | } 470 | return "" 471 | } 472 | 473 | func (x *Target) GetDstPort() int32 { 474 | if x != nil { 475 | return x.DstPort 476 | } 477 | return 0 478 | } 479 | 480 | type TargetList struct { 481 | state protoimpl.MessageState 482 | sizeCache protoimpl.SizeCache 483 | unknownFields protoimpl.UnknownFields 484 | 485 | Target []*Target `protobuf:"bytes,1,rep,name=target,proto3" json:"target,omitempty"` 486 | } 487 | 488 | func (x *TargetList) Reset() { 489 | *x = TargetList{} 490 | if protoimpl.UnsafeEnabled { 491 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[7] 492 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 493 | ms.StoreMessageInfo(mi) 494 | } 495 | } 496 | 497 | func (x *TargetList) String() string { 498 | return protoimpl.X.MessageStringOf(x) 499 | } 500 | 501 | func (*TargetList) ProtoMessage() {} 502 | 503 | func (x *TargetList) ProtoReflect() protoreflect.Message { 504 | mi := &file_internal_loadbalancer_loadbalancer_proto_msgTypes[7] 505 | if protoimpl.UnsafeEnabled && x != nil { 506 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 507 | if ms.LoadMessageInfo() == nil { 508 | ms.StoreMessageInfo(mi) 509 | } 510 | return ms 511 | } 512 | return mi.MessageOf(x) 513 | } 514 | 515 | // Deprecated: Use TargetList.ProtoReflect.Descriptor instead. 516 | func (*TargetList) Descriptor() ([]byte, []int) { 517 | return file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP(), []int{7} 518 | } 519 | 520 | func (x *TargetList) GetTarget() []*Target { 521 | if x != nil { 522 | return x.Target 523 | } 524 | return nil 525 | } 526 | 527 | var File_internal_loadbalancer_loadbalancer_proto protoreflect.FileDescriptor 528 | 529 | var file_internal_loadbalancer_loadbalancer_proto_rawDesc = []byte{ 530 | 0x0a, 0x28, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6c, 0x6f, 0x61, 0x64, 0x62, 531 | 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2f, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 532 | 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6c, 0x6f, 0x61, 0x64, 533 | 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 534 | 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 535 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x35, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 536 | 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x43, 0x6f, 537 | 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 538 | 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x26, 0x0a, 0x10, 539 | 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 540 | 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 541 | 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 542 | 0x61, 0x6e, 0x63, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 543 | 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 544 | 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 545 | 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x4c, 0x0a, 546 | 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 547 | 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 548 | 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 549 | 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 550 | 0x72, 0x79, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x1a, 0x54, 0x0a, 0x0c, 0x54, 551 | 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 552 | 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 553 | 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6c, 554 | 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x54, 0x61, 0x72, 0x67, 555 | 0x65, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 556 | 0x01, 0x22, 0x52, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x42, 557 | 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 558 | 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x07, 0x69, 559 | 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 560 | 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x88, 0x01, 0x01, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x69, 0x70, 561 | 0x5f, 0x61, 0x64, 0x64, 0x72, 0x22, 0x73, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x54, 0x61, 0x72, 0x67, 562 | 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6c, 0x62, 0x5f, 563 | 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x62, 0x4e, 0x61, 564 | 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x72, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 565 | 0x01, 0x28, 0x05, 0x52, 0x07, 0x73, 0x72, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x06, 566 | 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6c, 567 | 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x54, 0x61, 0x72, 0x67, 568 | 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x73, 0x0a, 0x10, 0x41, 0x64, 569 | 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 570 | 0x0a, 0x07, 0x6c, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 571 | 0x06, 0x6c, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x72, 0x63, 0x50, 0x6f, 572 | 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x73, 0x72, 0x63, 0x50, 0x6f, 0x72, 573 | 0x74, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 574 | 0x0b, 0x32, 0x14, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 575 | 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 576 | 0x6c, 0x0a, 0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x32, 0x0a, 0x08, 0x70, 0x72, 0x6f, 577 | 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6c, 0x6f, 578 | 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 579 | 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x14, 0x0a, 580 | 0x05, 0x64, 0x73, 0x74, 0x49, 0x50, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64, 0x73, 581 | 0x74, 0x49, 0x50, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 582 | 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x64, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x3a, 0x0a, 583 | 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x06, 0x74, 584 | 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6c, 0x6f, 585 | 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 586 | 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2a, 0x1c, 0x0a, 0x08, 0x50, 0x72, 0x6f, 587 | 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x00, 0x12, 0x07, 588 | 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x01, 0x32, 0xd3, 0x03, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 589 | 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x53, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4c, 590 | 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x73, 0x12, 0x16, 0x2e, 0x67, 591 | 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 592 | 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x25, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 593 | 0x63, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 594 | 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x30, 0x01, 0x12, 0x58, 0x0a, 595 | 0x0f, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 596 | 0x12, 0x1e, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 597 | 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 598 | 0x1a, 0x25, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 599 | 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 600 | 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x51, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 601 | 0x65, 0x12, 0x20, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 602 | 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 603 | 0x63, 0x65, 0x72, 0x1a, 0x25, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 604 | 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x49, 605 | 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 606 | 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 607 | 0x63, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 608 | 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x13, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 609 | 0x63, 0x65, 0x72, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x40, 0x0a, 0x09, 0x41, 0x64, 0x64, 610 | 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1e, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 611 | 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 612 | 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 613 | 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x40, 0x0a, 0x09, 0x44, 614 | 0x65, 0x6c, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1e, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 615 | 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x44, 0x65, 0x6c, 0x54, 0x61, 0x72, 0x67, 0x65, 616 | 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x62, 617 | 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x43, 0x5a, 618 | 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x6f, 0x72, 619 | 0x6f, 0x6b, 0x6d, 0x61, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x78, 0x6d, 0x6f, 0x78, 0x2d, 0x63, 0x6c, 620 | 0x6f, 0x75, 0x64, 0x2d, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x74, 621 | 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 622 | 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 623 | } 624 | 625 | var ( 626 | file_internal_loadbalancer_loadbalancer_proto_rawDescOnce sync.Once 627 | file_internal_loadbalancer_loadbalancer_proto_rawDescData = file_internal_loadbalancer_loadbalancer_proto_rawDesc 628 | ) 629 | 630 | func file_internal_loadbalancer_loadbalancer_proto_rawDescGZIP() []byte { 631 | file_internal_loadbalancer_loadbalancer_proto_rawDescOnce.Do(func() { 632 | file_internal_loadbalancer_loadbalancer_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_loadbalancer_loadbalancer_proto_rawDescData) 633 | }) 634 | return file_internal_loadbalancer_loadbalancer_proto_rawDescData 635 | } 636 | 637 | var file_internal_loadbalancer_loadbalancer_proto_enumTypes = make([]protoimpl.EnumInfo, 1) 638 | var file_internal_loadbalancer_loadbalancer_proto_msgTypes = make([]protoimpl.MessageInfo, 9) 639 | var file_internal_loadbalancer_loadbalancer_proto_goTypes = []interface{}{ 640 | (Protocol)(0), // 0: loadbalancer.Protocol 641 | (*Error)(nil), // 1: loadbalancer.Error 642 | (*LoadBalancerName)(nil), // 2: loadbalancer.LoadBalancerName 643 | (*LoadBalancerInformation)(nil), // 3: loadbalancer.LoadBalancerInformation 644 | (*CreateLoadBalancer)(nil), // 4: loadbalancer.CreateLoadBalancer 645 | (*DelTargetRequest)(nil), // 5: loadbalancer.DelTargetRequest 646 | (*AddTargetRequest)(nil), // 6: loadbalancer.AddTargetRequest 647 | (*Target)(nil), // 7: loadbalancer.Target 648 | (*TargetList)(nil), // 8: loadbalancer.TargetList 649 | nil, // 9: loadbalancer.LoadBalancerInformation.TargetsEntry 650 | (*emptypb.Empty)(nil), // 10: google.protobuf.Empty 651 | } 652 | var file_internal_loadbalancer_loadbalancer_proto_depIdxs = []int32{ 653 | 9, // 0: loadbalancer.LoadBalancerInformation.targets:type_name -> loadbalancer.LoadBalancerInformation.TargetsEntry 654 | 7, // 1: loadbalancer.DelTargetRequest.target:type_name -> loadbalancer.Target 655 | 7, // 2: loadbalancer.AddTargetRequest.target:type_name -> loadbalancer.Target 656 | 0, // 3: loadbalancer.Target.protocol:type_name -> loadbalancer.Protocol 657 | 7, // 4: loadbalancer.TargetList.target:type_name -> loadbalancer.Target 658 | 8, // 5: loadbalancer.LoadBalancerInformation.TargetsEntry.value:type_name -> loadbalancer.TargetList 659 | 10, // 6: loadbalancer.LoadBalancer.GetLoadBalancers:input_type -> google.protobuf.Empty 660 | 2, // 7: loadbalancer.LoadBalancer.GetLoadBalancer:input_type -> loadbalancer.LoadBalancerName 661 | 4, // 8: loadbalancer.LoadBalancer.Create:input_type -> loadbalancer.CreateLoadBalancer 662 | 2, // 9: loadbalancer.LoadBalancer.Delete:input_type -> loadbalancer.LoadBalancerName 663 | 6, // 10: loadbalancer.LoadBalancer.AddTarget:input_type -> loadbalancer.AddTargetRequest 664 | 5, // 11: loadbalancer.LoadBalancer.DelTarget:input_type -> loadbalancer.DelTargetRequest 665 | 3, // 12: loadbalancer.LoadBalancer.GetLoadBalancers:output_type -> loadbalancer.LoadBalancerInformation 666 | 3, // 13: loadbalancer.LoadBalancer.GetLoadBalancer:output_type -> loadbalancer.LoadBalancerInformation 667 | 3, // 14: loadbalancer.LoadBalancer.Create:output_type -> loadbalancer.LoadBalancerInformation 668 | 1, // 15: loadbalancer.LoadBalancer.Delete:output_type -> loadbalancer.Error 669 | 1, // 16: loadbalancer.LoadBalancer.AddTarget:output_type -> loadbalancer.Error 670 | 1, // 17: loadbalancer.LoadBalancer.DelTarget:output_type -> loadbalancer.Error 671 | 12, // [12:18] is the sub-list for method output_type 672 | 6, // [6:12] is the sub-list for method input_type 673 | 6, // [6:6] is the sub-list for extension type_name 674 | 6, // [6:6] is the sub-list for extension extendee 675 | 0, // [0:6] is the sub-list for field type_name 676 | } 677 | 678 | func init() { file_internal_loadbalancer_loadbalancer_proto_init() } 679 | func file_internal_loadbalancer_loadbalancer_proto_init() { 680 | if File_internal_loadbalancer_loadbalancer_proto != nil { 681 | return 682 | } 683 | if !protoimpl.UnsafeEnabled { 684 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 685 | switch v := v.(*Error); i { 686 | case 0: 687 | return &v.state 688 | case 1: 689 | return &v.sizeCache 690 | case 2: 691 | return &v.unknownFields 692 | default: 693 | return nil 694 | } 695 | } 696 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 697 | switch v := v.(*LoadBalancerName); i { 698 | case 0: 699 | return &v.state 700 | case 1: 701 | return &v.sizeCache 702 | case 2: 703 | return &v.unknownFields 704 | default: 705 | return nil 706 | } 707 | } 708 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 709 | switch v := v.(*LoadBalancerInformation); i { 710 | case 0: 711 | return &v.state 712 | case 1: 713 | return &v.sizeCache 714 | case 2: 715 | return &v.unknownFields 716 | default: 717 | return nil 718 | } 719 | } 720 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { 721 | switch v := v.(*CreateLoadBalancer); i { 722 | case 0: 723 | return &v.state 724 | case 1: 725 | return &v.sizeCache 726 | case 2: 727 | return &v.unknownFields 728 | default: 729 | return nil 730 | } 731 | } 732 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { 733 | switch v := v.(*DelTargetRequest); i { 734 | case 0: 735 | return &v.state 736 | case 1: 737 | return &v.sizeCache 738 | case 2: 739 | return &v.unknownFields 740 | default: 741 | return nil 742 | } 743 | } 744 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { 745 | switch v := v.(*AddTargetRequest); i { 746 | case 0: 747 | return &v.state 748 | case 1: 749 | return &v.sizeCache 750 | case 2: 751 | return &v.unknownFields 752 | default: 753 | return nil 754 | } 755 | } 756 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { 757 | switch v := v.(*Target); i { 758 | case 0: 759 | return &v.state 760 | case 1: 761 | return &v.sizeCache 762 | case 2: 763 | return &v.unknownFields 764 | default: 765 | return nil 766 | } 767 | } 768 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { 769 | switch v := v.(*TargetList); i { 770 | case 0: 771 | return &v.state 772 | case 1: 773 | return &v.sizeCache 774 | case 2: 775 | return &v.unknownFields 776 | default: 777 | return nil 778 | } 779 | } 780 | } 781 | file_internal_loadbalancer_loadbalancer_proto_msgTypes[3].OneofWrappers = []interface{}{} 782 | type x struct{} 783 | out := protoimpl.TypeBuilder{ 784 | File: protoimpl.DescBuilder{ 785 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 786 | RawDescriptor: file_internal_loadbalancer_loadbalancer_proto_rawDesc, 787 | NumEnums: 1, 788 | NumMessages: 9, 789 | NumExtensions: 0, 790 | NumServices: 1, 791 | }, 792 | GoTypes: file_internal_loadbalancer_loadbalancer_proto_goTypes, 793 | DependencyIndexes: file_internal_loadbalancer_loadbalancer_proto_depIdxs, 794 | EnumInfos: file_internal_loadbalancer_loadbalancer_proto_enumTypes, 795 | MessageInfos: file_internal_loadbalancer_loadbalancer_proto_msgTypes, 796 | }.Build() 797 | File_internal_loadbalancer_loadbalancer_proto = out.File 798 | file_internal_loadbalancer_loadbalancer_proto_rawDesc = nil 799 | file_internal_loadbalancer_loadbalancer_proto_goTypes = nil 800 | file_internal_loadbalancer_loadbalancer_proto_depIdxs = nil 801 | } 802 | --------------------------------------------------------------------------------