├── PROJECT ├── docs └── pics │ ├── current-lb.png │ ├── shared-lb.png │ └── 4-dimensions.png ├── demos ├── eks │ ├── crs │ │ ├── cr-tcp1.yaml │ │ ├── cr-tcp2.yaml │ │ └── cr-tcp3.yaml │ ├── deployments │ │ ├── tcp1-deploy.yaml │ │ ├── tcp2-deploy.yaml │ │ └── tcp3-deploy.yaml │ └── demo.sh ├── gke │ ├── crs │ │ ├── cr-tcp1.yaml │ │ ├── cr-tcp2.yaml │ │ ├── cr-tcp3.yaml │ │ └── cr-tcp4.yaml │ ├── deployments │ │ ├── tcp1-deploy.yaml │ │ ├── tcp2-deploy.yaml │ │ ├── tcp3-deploy.yaml │ │ └── tcp4-deploy.yaml │ └── demo.sh ├── iks │ ├── crs │ │ ├── cr-tcp.yaml │ │ ├── cr-udp.yaml │ │ └── cr-tcp-random-port.yaml │ ├── deployments │ │ ├── tcp-deploy.yaml │ │ └── udp-deploy.yaml │ └── demo.sh ├── aks │ ├── crs │ │ ├── cr-tcp1-4000.yaml │ │ ├── cr-tcp2-4000.yaml │ │ └── cr-tcp-random.yaml │ ├── deployments │ │ ├── tcp1-deploy.yaml │ │ └── tcp2-deploy.yaml │ └── demo.sh └── demoscript ├── config ├── samples │ ├── kubecon_v1alpha1_sharedlb.yaml │ ├── kubecon_v1alpha1_sharedlb1.yaml │ ├── kubecon_v1alpha1_sharedlb11.yaml │ ├── kubecon_v1alpha1_sharedlb111.yaml │ ├── kubecon_v1alpha1_sharedlb2.yaml │ └── kubecon_v1alpha1_sharedlb3.yaml ├── default │ ├── manager_image_patch.yaml │ └── kustomization.yaml ├── rbac │ ├── rbac_role_binding.yaml │ └── rbac_role.yaml ├── eks │ └── placeholder-deploy.yaml ├── crds │ └── kubecon_v1alpha1_sharedlb.yaml └── manager │ └── manager.yaml ├── .gitignore ├── Dockerfile ├── hack └── boilerplate.go.txt ├── pkg ├── apis │ ├── kubecon │ │ ├── group.go │ │ └── v1alpha1 │ │ │ ├── doc.go │ │ │ ├── v1alpha1_suite_test.go │ │ │ ├── register.go │ │ │ ├── sharedlb_types_test.go │ │ │ ├── sharedlb_types.go │ │ │ └── zz_generated.deepcopy.go │ ├── addtoscheme_kubecon_v1alpha1.go │ └── apis.go ├── controller │ ├── add_sharedlb.go │ ├── controller.go │ └── sharedlb │ │ ├── sharedlb_controller_suite_test.go │ │ ├── sharedlb_controller_test.go │ │ └── sharedlb_controller.go ├── webhook │ └── webhook.go └── providers │ ├── utils_test.go │ ├── utils.go │ ├── base.go │ ├── local.go │ ├── aks_test.go │ ├── iks.go │ ├── gke.go │ ├── eks.go │ └── aks.go ├── Makefile ├── cmd └── manager │ └── main.go ├── README.md ├── go.mod └── go.sum /PROJECT: -------------------------------------------------------------------------------- 1 | version: "1" 2 | domain: k8s.io 3 | repo: github.com/Huang-Wei/shared-loadbalancer 4 | -------------------------------------------------------------------------------- /docs/pics/current-lb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Huang-Wei/shared-loadbalancer/HEAD/docs/pics/current-lb.png -------------------------------------------------------------------------------- /docs/pics/shared-lb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Huang-Wei/shared-loadbalancer/HEAD/docs/pics/shared-lb.png -------------------------------------------------------------------------------- /docs/pics/4-dimensions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Huang-Wei/shared-loadbalancer/HEAD/docs/pics/4-dimensions.png -------------------------------------------------------------------------------- /demos/eks/crs/cr-tcp1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp1 5 | spec: 6 | ports: 7 | - port: 4001 8 | targetPort: 4000 9 | selector: 10 | app: tcp1 11 | -------------------------------------------------------------------------------- /demos/eks/crs/cr-tcp2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp2 5 | spec: 6 | ports: 7 | - port: 4002 8 | targetPort: 4000 9 | selector: 10 | app: tcp2 11 | -------------------------------------------------------------------------------- /demos/eks/crs/cr-tcp3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp3 5 | spec: 6 | ports: 7 | - port: 4003 8 | targetPort: 4000 9 | selector: 10 | app: tcp3 11 | -------------------------------------------------------------------------------- /demos/gke/crs/cr-tcp1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp1 5 | spec: 6 | ports: 7 | - port: 4001 8 | targetPort: 4000 9 | selector: 10 | app: tcp1 11 | -------------------------------------------------------------------------------- /demos/gke/crs/cr-tcp2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp2 5 | spec: 6 | ports: 7 | - port: 4002 8 | targetPort: 4000 9 | selector: 10 | app: tcp2 11 | -------------------------------------------------------------------------------- /demos/gke/crs/cr-tcp3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp3 5 | spec: 6 | ports: 7 | - port: 4003 8 | targetPort: 4000 9 | selector: 10 | app: tcp3 11 | -------------------------------------------------------------------------------- /demos/gke/crs/cr-tcp4.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp4 5 | spec: 6 | ports: 7 | - port: 4004 8 | targetPort: 4000 9 | selector: 10 | app: tcp4 11 | -------------------------------------------------------------------------------- /demos/iks/crs/cr-tcp.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp 5 | spec: 6 | ports: 7 | - port: 4001 8 | targetPort: 4000 9 | selector: 10 | app: tcp 11 | -------------------------------------------------------------------------------- /demos/aks/crs/cr-tcp1-4000.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp1 5 | spec: 6 | ports: 7 | - port: 4000 8 | targetPort: 4000 9 | selector: 10 | app: tcp1 11 | -------------------------------------------------------------------------------- /demos/aks/crs/cr-tcp2-4000.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp2 5 | spec: 6 | ports: 7 | - port: 4000 8 | targetPort: 4000 9 | selector: 10 | app: tcp2 11 | -------------------------------------------------------------------------------- /demos/aks/crs/cr-tcp-random.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp-random 5 | spec: 6 | ports: 7 | # - port: 4000 8 | - targetPort: 4000 9 | selector: 10 | app: tcp 11 | -------------------------------------------------------------------------------- /demos/iks/crs/cr-udp.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-udp 5 | spec: 6 | ports: 7 | - port: 5001 8 | targetPort: 5000 9 | protocol: UDP 10 | selector: 11 | app: udp 12 | -------------------------------------------------------------------------------- /demos/iks/crs/cr-tcp-random-port.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | name: sharedlb-tcp-random-port 5 | spec: 6 | ports: 7 | # - port: 4001 8 | - targetPort: 4000 9 | selector: 10 | app: tcp 11 | -------------------------------------------------------------------------------- /config/samples/kubecon_v1alpha1_sharedlb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: sharedlb-sample 7 | spec: 8 | ports: 9 | - port: 8080 10 | targetPort: 80 11 | selector: 12 | app: nginx 13 | -------------------------------------------------------------------------------- /config/samples/kubecon_v1alpha1_sharedlb1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: sharedlb-sample1 7 | spec: 8 | ports: 9 | - port: 8081 10 | targetPort: 80 11 | selector: 12 | app: nginx 13 | -------------------------------------------------------------------------------- /config/samples/kubecon_v1alpha1_sharedlb11.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: sharedlb-sample11 7 | spec: 8 | ports: 9 | - port: 8081 10 | targetPort: 80 11 | selector: 12 | app: nginx 13 | -------------------------------------------------------------------------------- /config/samples/kubecon_v1alpha1_sharedlb111.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: sharedlb-sample111 7 | spec: 8 | ports: 9 | - port: 8081 10 | targetPort: 80 11 | selector: 12 | app: nginx 13 | -------------------------------------------------------------------------------- /config/samples/kubecon_v1alpha1_sharedlb2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: sharedlb-sample2 7 | spec: 8 | ports: 9 | - port: 8082 10 | targetPort: 80 11 | selector: 12 | app: nginx 13 | -------------------------------------------------------------------------------- /config/samples/kubecon_v1alpha1_sharedlb3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubecon.k8s.io/v1alpha1 2 | kind: SharedLB 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: sharedlb-sample3 7 | spec: 8 | ports: 9 | - port: 8083 10 | targetPort: 80 11 | selector: 12 | app: nginx 13 | -------------------------------------------------------------------------------- /config/default/manager_image_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | # Change the value of image field below to your controller image URL 11 | - image: IMAGE_URL 12 | name: manager 13 | -------------------------------------------------------------------------------- /config/rbac/rbac_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | creationTimestamp: null 5 | name: manager-rolebinding 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: manager-role 10 | subjects: 11 | - kind: ServiceAccount 12 | name: default 13 | namespace: system 14 | -------------------------------------------------------------------------------- /demos/iks/deployments/tcp-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp 14 | spec: 15 | containers: 16 | - name: tcp 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/iks/deployments/udp-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: udp-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: udp 10 | template: 11 | metadata: 12 | labels: 13 | app: udp 14 | spec: 15 | containers: 16 | - name: udp 17 | image: hweicdl/netcat-udp:v0.1.0 18 | ports: 19 | - containerPort: 5000 20 | -------------------------------------------------------------------------------- /demos/aks/deployments/tcp1-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp1-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp1 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp1 14 | spec: 15 | containers: 16 | - name: tcp1 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/aks/deployments/tcp2-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp2-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp2 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp2 14 | spec: 15 | containers: 16 | - name: tcp2 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/eks/deployments/tcp1-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp1-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp1 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp1 14 | spec: 15 | containers: 16 | - name: tcp1 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/eks/deployments/tcp2-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp2-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp2 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp2 14 | spec: 15 | containers: 16 | - name: tcp2 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/eks/deployments/tcp3-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp3-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp3 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp3 14 | spec: 15 | containers: 16 | - name: tcp3 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/gke/deployments/tcp1-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp1-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp1 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp1 14 | spec: 15 | containers: 16 | - name: tcp1 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/gke/deployments/tcp2-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp2-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp2 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp2 14 | spec: 15 | containers: 16 | - name: tcp2 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/gke/deployments/tcp3-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp3-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp3 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp3 14 | spec: 15 | containers: 16 | - name: tcp3 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /demos/gke/deployments/tcp4-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: tcp4-deploy 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: tcp4 10 | template: 11 | metadata: 12 | labels: 13 | app: tcp4 14 | spec: 15 | containers: 16 | - name: tcp4 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | ports: 19 | - containerPort: 4000 20 | -------------------------------------------------------------------------------- /config/eks/placeholder-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: lb-placeholder 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: lb-placeholder 10 | template: 11 | metadata: 12 | labels: 13 | app: lb-placeholder 14 | spec: 15 | containers: 16 | - name: lb-placeholder 17 | image: hweicdl/netcat-tcp:v0.1.0 18 | command: ["nc"] 19 | args: ["-kl", "33333"] # no verbose 20 | ports: 21 | - containerPort: 33333 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Doit ignore files 2 | cmds 3 | out 4 | 5 | # Azure local testing config file 6 | .env 7 | 8 | # Binaries for programs and plugins 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | bin 15 | 16 | # Test binary, build with `go test -c` 17 | *.test 18 | 19 | # Output of the go coverage tool, specifically when used with LiteIDE 20 | *.out 21 | 22 | # Kubernetes Generated files - skip generated files, except for vendored files 23 | 24 | vendor/ 25 | !vendor/**/zz_generated.* 26 | 27 | # editor and IDE paraphernalia 28 | .idea 29 | *.swp 30 | *.swo 31 | *~ 32 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.10.3 as builder 3 | 4 | # Copy in the go src 5 | WORKDIR /go/src/github.com/Huang-Wei/shared-loadbalancer 6 | COPY pkg/ pkg/ 7 | COPY cmd/ cmd/ 8 | COPY vendor/ vendor/ 9 | 10 | # Build 11 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager github.com/Huang-Wei/shared-loadbalancer/cmd/manager 12 | 13 | # Copy the controller-manager into a thin image 14 | FROM ubuntu:latest 15 | WORKDIR /root/ 16 | COPY --from=builder /go/src/github.com/Huang-Wei/shared-loadbalancer/manager . 17 | ENTRYPOINT ["./manager"] 18 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /pkg/apis/kubecon/group.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package kubecon contains kubecon API versions 18 | package kubecon 19 | -------------------------------------------------------------------------------- /pkg/controller/add_sharedlb.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controller 18 | 19 | import ( 20 | "github.com/Huang-Wei/shared-loadbalancer/pkg/controller/sharedlb" 21 | ) 22 | 23 | func init() { 24 | // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. 25 | AddToManagerFuncs = append(AddToManagerFuncs, sharedlb.Add) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/apis/kubecon/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the kubecon v1alpha1 API group 18 | // +k8s:openapi-gen=true 19 | // +k8s:deepcopy-gen=package,register 20 | // +k8s:conversion-gen=github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon 21 | // +k8s:defaulter-gen=TypeMeta 22 | // +groupName=kubecon.k8s.io 23 | package v1alpha1 24 | -------------------------------------------------------------------------------- /pkg/apis/addtoscheme_kubecon_v1alpha1.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apis 18 | 19 | import ( 20 | "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 21 | ) 22 | 23 | func init() { 24 | // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back 25 | AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme) 26 | } 27 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: shared-loadbalancer-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: shared-loadbalancer- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | # Each entry in this list must resolve to an existing 16 | # resource definition in YAML. These are the resource 17 | # files that kustomize reads, modifies and emits as a 18 | # YAML string, with resources separated by document 19 | # markers ("---"). 20 | resources: 21 | - ../rbac/rbac_role.yaml 22 | - ../rbac/rbac_role_binding.yaml 23 | - ../manager/manager.yaml 24 | 25 | patches: 26 | - manager_image_patch.yaml 27 | 28 | vars: 29 | - name: WEBHOOK_SECRET_NAME 30 | objref: 31 | kind: Secret 32 | name: webhook-server-secret 33 | apiVersion: v1 34 | -------------------------------------------------------------------------------- /pkg/controller/controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controller 18 | 19 | import ( 20 | "sigs.k8s.io/controller-runtime/pkg/manager" 21 | ) 22 | 23 | // AddToManagerFuncs is a list of functions to add all Controllers to the Manager 24 | var AddToManagerFuncs []func(manager.Manager) error 25 | 26 | // AddToManager adds all Controllers to the Manager 27 | func AddToManager(m manager.Manager) error { 28 | for _, f := range AddToManagerFuncs { 29 | if err := f(m); err != nil { 30 | return err 31 | } 32 | } 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /config/rbac/rbac_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | creationTimestamp: null 5 | name: manager-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - services 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - kubecon.k8s.io 21 | resources: 22 | - sharedlbs 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - admissionregistration.k8s.io 33 | resources: 34 | - mutatingwebhookconfigurations 35 | - validatingwebhookconfigurations 36 | verbs: 37 | - get 38 | - list 39 | - watch 40 | - create 41 | - update 42 | - patch 43 | - delete 44 | - apiGroups: 45 | - "" 46 | resources: 47 | - secrets 48 | verbs: 49 | - get 50 | - list 51 | - watch 52 | - create 53 | - update 54 | - patch 55 | - delete 56 | - apiGroups: 57 | - "" 58 | resources: 59 | - services 60 | verbs: 61 | - get 62 | - list 63 | - watch 64 | - create 65 | - update 66 | - patch 67 | - delete 68 | -------------------------------------------------------------------------------- /pkg/apis/apis.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Generate deepcopy for apis 18 | //go:generate go run ../../vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h ../../hack/boilerplate.go.txt 19 | 20 | // Package apis contains Kubernetes API groups. 21 | package apis 22 | 23 | import ( 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // AddToSchemes may be used to add all resources defined in the project to a Scheme 28 | var AddToSchemes runtime.SchemeBuilder 29 | 30 | // AddToScheme adds all Resources to the Scheme 31 | func AddToScheme(s *runtime.Scheme) error { 32 | return AddToSchemes.AddToScheme(s) 33 | } 34 | -------------------------------------------------------------------------------- /demos/gke/demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "$0")/../demoscript 4 | source ~/.gke 5 | 6 | comment This is a demo on Google Kubernetes Engine 7 | 8 | doit source ~/.gke 9 | doit echo "\$KUBECONFIG" 10 | 11 | doit kubectl get node 12 | doit kubectl version --short 13 | 14 | doit kubectl get svc,pod 15 | 16 | comment We are going to create some deployments 17 | 18 | doit kubectl create -f demos/gke/deployments/ 19 | 20 | comment Custom Resource Definition has been pre-created 21 | doit kubectl get crd 22 | doit kubectl get slb 23 | 24 | comment We are going to create 4 SharedLB CRs in parallel 25 | doit kubectl create -f demos/gke/crs 26 | 27 | comment So are you expected to see FOUR LBs being created, or just ONE? 28 | doit kubectl get svc 29 | doit kubectl get slb 30 | 31 | comment GKE LB needs ~1min to be created 32 | doit kubectl get svc 33 | doit kubectl get slb 34 | 35 | comment Use nc to try connecting 36 | out=$(kubectl get slb) 37 | doit nc -zv $(echo "$out" | grep sharedlb-tcp1 | awk '{print $2}') $(echo "$out" | grep sharedlb-tcp1 | awk '{print $3}') 38 | doit nc -zv $(echo "$out" | grep sharedlb-tcp4 | awk '{print $2}') $(echo "$out" | grep sharedlb-tcp4 | awk '{print $3}') 39 | 40 | # comment Cleanup 41 | 42 | # doit kubectl delete deploy --all 43 | # doit kubectl delete slb --all 44 | # doit kubectl delete svc -l=lb-template= 45 | 46 | comment ~End~ -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= controller:latest 4 | 5 | all: test manager 6 | 7 | # Run tests 8 | test: generate fmt vet manifests 9 | go test ./pkg/... ./cmd/... -coverprofile cover.out 10 | 11 | # Build manager binary 12 | manager: generate fmt vet 13 | go build -o bin/manager github.com/Huang-Wei/shared-loadbalancer/cmd/manager 14 | 15 | # Run against the configured Kubernetes cluster in ~/.kube/config 16 | run: generate fmt vet 17 | go run ./cmd/manager/main.go 18 | 19 | # Install CRDs into a cluster 20 | install: manifests 21 | kubectl apply -f config/crds 22 | 23 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 24 | deploy: manifests 25 | kubectl apply -f config/crds 26 | kustomize build config/default | kubectl apply -f - 27 | 28 | # Generate manifests e.g. CRD, RBAC etc. 29 | manifests: 30 | go run vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go all 31 | 32 | # Run go fmt against code 33 | fmt: 34 | go fmt ./pkg/... ./cmd/... 35 | 36 | # Run go vet against code 37 | vet: 38 | go vet ./pkg/... ./cmd/... 39 | 40 | # Generate code 41 | generate: 42 | go generate ./pkg/... ./cmd/... 43 | 44 | # Build the docker image 45 | docker-build: test 46 | docker build . -t ${IMG} 47 | @echo "updating kustomize image patch file for manager resource" 48 | sed -i'' -e 's@image: .*@image: '"${IMG}"'@' ./config/default/manager_image_patch.yaml 49 | 50 | # Push the docker image 51 | docker-push: 52 | docker push ${IMG} 53 | -------------------------------------------------------------------------------- /pkg/webhook/webhook.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package webhook 18 | 19 | import ( 20 | "sigs.k8s.io/controller-runtime/pkg/manager" 21 | ) 22 | 23 | // AddToManagerFuncs is a list of functions to add all Controllers to the Manager 24 | var AddToManagerFuncs []func(manager.Manager) error 25 | 26 | // AddToManager adds all Controllers to the Manager 27 | // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations;validatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete 28 | // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete 29 | // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete 30 | func AddToManager(m manager.Manager) error { 31 | for _, f := range AddToManagerFuncs { 32 | if err := f(m); err != nil { 33 | return err 34 | } 35 | } 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /pkg/apis/kubecon/v1alpha1/v1alpha1_suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "log" 21 | "os" 22 | "path/filepath" 23 | "testing" 24 | 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/rest" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/envtest" 29 | ) 30 | 31 | var cfg *rest.Config 32 | var c client.Client 33 | 34 | func TestMain(m *testing.M) { 35 | t := &envtest.Environment{ 36 | CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crds")}, 37 | } 38 | 39 | err := SchemeBuilder.AddToScheme(scheme.Scheme) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | if cfg, err = t.Start(); err != nil { 45 | log.Fatal(err) 46 | } 47 | 48 | if c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}); err != nil { 49 | log.Fatal(err) 50 | } 51 | 52 | code := m.Run() 53 | t.Stop() 54 | os.Exit(code) 55 | } 56 | -------------------------------------------------------------------------------- /config/crds/kubecon_v1alpha1_sharedlb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | controller-tools.k8s.io: "1.0" 7 | name: sharedlbs.kubecon.k8s.io 8 | spec: 9 | group: kubecon.k8s.io 10 | names: 11 | kind: SharedLB 12 | plural: sharedlbs 13 | shortNames: 14 | - slb 15 | additionalPrinterColumns: 16 | - name: External-IP 17 | type: string 18 | JSONPath: .status.loadBalancer.ingress[*].* 19 | - name: Port 20 | type: string 21 | JSONPath: .spec.ports[*].port 22 | # can't combine protocol info along with "Port" column until 23 | # https://github.com/kubernetes/kubernetes/issues/67268 gets implemented 24 | - name: Protocol 25 | type: string 26 | JSONPath: .spec.ports[*].protocol 27 | - name: Ref 28 | type: string 29 | JSONPath: .status.ref 30 | scope: Namespaced 31 | validation: 32 | openAPIV3Schema: 33 | properties: 34 | apiVersion: 35 | type: string 36 | kind: 37 | type: string 38 | metadata: 39 | type: object 40 | spec: 41 | properties: 42 | loadBalancerIP: 43 | type: string 44 | ports: 45 | items: 46 | type: object 47 | type: array 48 | selector: 49 | type: object 50 | type: object 51 | status: 52 | properties: 53 | loadBalancer: 54 | type: object 55 | type: object 56 | version: v1alpha1 57 | status: 58 | acceptedNames: 59 | kind: "" 60 | plural: "" 61 | conditions: [] 62 | storedVersions: [] 63 | -------------------------------------------------------------------------------- /pkg/apis/kubecon/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // NOTE: Boilerplate only. Ignore this file. 18 | 19 | // Package v1alpha1 contains API Schema definitions for the kubecon v1alpha1 API group 20 | // +k8s:openapi-gen=true 21 | // +k8s:deepcopy-gen=package,register 22 | // +k8s:conversion-gen=github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon 23 | // +k8s:defaulter-gen=TypeMeta 24 | // +groupName=kubecon.k8s.io 25 | package v1alpha1 26 | 27 | import ( 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "sigs.k8s.io/controller-runtime/pkg/runtime/scheme" 30 | ) 31 | 32 | var ( 33 | // SchemeGroupVersion is group version used to register these objects 34 | SchemeGroupVersion = schema.GroupVersion{Group: "kubecon.k8s.io", Version: "v1alpha1"} 35 | 36 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 37 | SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} 38 | 39 | // AddToScheme is required by pkg/client/... 40 | AddToScheme = SchemeBuilder.AddToScheme 41 | ) 42 | 43 | // Resource is required by pkg/client/listers/... 44 | func Resource(resource string) schema.GroupResource { 45 | return SchemeGroupVersion.WithResource(resource).GroupResource() 46 | } 47 | -------------------------------------------------------------------------------- /pkg/providers/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "os" 21 | "testing" 22 | ) 23 | 24 | func TestGetEnvValInt(t *testing.T) { 25 | tests := []struct { 26 | name string 27 | envKey string 28 | defaultVal int 29 | want int 30 | preSetup func() 31 | }{ 32 | { 33 | name: "env variable not exist", 34 | envKey: "CAPACITYTEST", 35 | defaultVal: 2, 36 | want: 2, 37 | preSetup: func() {}, 38 | }, 39 | { 40 | name: "env variable exists but with a non-int value", 41 | envKey: "CAPACITYTEST", 42 | defaultVal: 2, 43 | want: 2, 44 | preSetup: func() { 45 | os.Setenv("CAPACITYTEST", "1a2b") 46 | }, 47 | }, 48 | { 49 | name: "env variable exists and with a int value", 50 | envKey: "CAPACITYTEST", 51 | defaultVal: 2, 52 | want: 10, 53 | preSetup: func() { 54 | os.Setenv("CAPACITYTEST", "10") 55 | }, 56 | }, 57 | } 58 | for _, tt := range tests { 59 | t.Run(tt.name, func(t *testing.T) { 60 | tt.preSetup() 61 | if got := GetEnvValInt(tt.envKey, tt.defaultVal); got != tt.want { 62 | t.Errorf("GetEnvValInt() = %v, want %v", got, tt.want) 63 | } 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /demos/aks/demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "$0")/../demoscript 4 | source ~/.aks 5 | 6 | comment This is a demo on Azure Kubernetes Service 7 | 8 | doit source ~/.aks 9 | doit echo "\$KUBECONFIG" 10 | 11 | doit kubectl get node 12 | doit kubectl version --short 13 | 14 | doit kubectl get svc,pod 15 | 16 | comment We are going to create some deployments 17 | doit kubectl create -f demos/aks/deployments/ 18 | 19 | comment Custom Resource Definition has been pre-created 20 | doit kubectl get crd 21 | doit kubectl get slb 22 | 23 | comment We are going to create some SharedLB CRs 24 | doit cat demos/aks/crs/cr-tcp1-4000.yaml 25 | doit kubectl create -f demos/aks/crs/cr-tcp1-4000.yaml 26 | 27 | doit kubectl get svc 28 | doit kubectl get slb 29 | 30 | comment AKS LB needs ~1 minute to be created 31 | 32 | doit kubectl get svc 33 | doit kubectl get slb 34 | 35 | comment What if another ShareLB CR also requests port 4000 36 | doit cat demos/aks/crs/cr-tcp2-4000.yaml 37 | doit kubectl create -f demos/aks/crs/cr-tcp2-4000.yaml 38 | 39 | doit kubectl get svc 40 | doit kubectl get slb 41 | 42 | comment AKS LB needs another ~1 minute to be created 43 | doit kubectl get svc 44 | doit kubectl get slb 45 | 46 | comment Use nc to try connecting 47 | out=$(kubectl get slb) 48 | doit nc -zv $(echo "$out" | grep sharedlb-tcp1 | awk '{print $2}') $(echo "$out" | grep sharedlb-tcp1 | awk '{print $3}') 49 | doit nc -zv $(echo "$out" | grep sharedlb-tcp2 | awk '{print $2}') $(echo "$out" | grep sharedlb-tcp2 | awk '{print $3}') 50 | 51 | comment Right now both LBs have spare capacity 52 | doit kubectl create -f demos/aks/crs/cr-tcp-random.yaml 53 | doit kubectl get svc 54 | doit kubectl get slb 55 | 56 | # comment Cleanup 57 | 58 | # doit kubectl delete deploy --all 59 | # doit kubectl delete slb --all 60 | # doit kubectl delete svc -l=lb-template= 61 | 62 | comment ~End~ -------------------------------------------------------------------------------- /demos/iks/demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "$0")/../demoscript 4 | source ~/.iks 5 | 6 | comment This is a demo on IBM Kubernetes Service 7 | 8 | doit source ~/.iks 9 | doit echo "\$KUBECONFIG" 10 | 11 | doit kubectl get node 12 | doit kubectl version --short 13 | 14 | doit kubectl get svc,pod 15 | 16 | comment We are going to create 2 deployments - one TCP and one UDP 17 | doit kubectl create -f demos/iks/deployments/ 18 | 19 | comment Custom Resource Definition has been pre-created 20 | doit kubectl get crd 21 | doit kubectl get slb 22 | 23 | comment We are going to create 2 SharedLB CRs 24 | doit cat demos/iks/crs/cr-tcp.yaml 25 | doit cat demos/iks/crs/cr-udp.yaml 26 | 27 | doit kubectl create -f demos/iks/crs/cr-tcp.yaml 28 | doit kubectl get svc 29 | doit kubectl get slb 30 | 31 | comment By default CAPACITY is 5 32 | doit kubectl create -f demos/iks/crs/cr-udp.yaml 33 | doit kubectl get svc 34 | doit kubectl get slb 35 | 36 | comment Use nc to try connecting 37 | 38 | doit "echo 'tcp-test $(date)' | nc $(kubectl get slb | grep sharedlb-tcp | awk '{print $2}') $(kubectl get slb sharedlb-tcp | grep sharedlb-tcp | awk '{print $3}')" 39 | doit "echo 'udp-test $(date)' | nc -w 1 -u $(kubectl get slb | grep sharedlb-udp | awk '{print $2}') $(kubectl get slb sharedlb-udp | grep sharedlb-udp | awk '{print $3}')" 40 | 41 | comment Checkout pod logs 42 | 43 | doit kubectl logs $(kubectl get po | grep tcp-deploy | awk '{print $1}') 44 | doit kubectl logs $(kubectl get po | grep udp-deploy | awk '{print $1}') 45 | 46 | comment What if I just want a random source port 47 | 48 | doit cat demos/iks/crs/cr-tcp-random-port.yaml 49 | doit kubectl create -f demos/iks/crs/cr-tcp-random-port.yaml 50 | doit kubectl get svc 51 | doit kubectl get slb 52 | 53 | # comment Cleanup 54 | 55 | # doit kubectl delete deploy --all 56 | # doit kubectl delete slb --all 57 | # doit kubectl delete svc -l=lb-template= 58 | 59 | comment ~End~ -------------------------------------------------------------------------------- /pkg/apis/kubecon/v1alpha1/sharedlb_types_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/onsi/gomega" 23 | "golang.org/x/net/context" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/types" 26 | ) 27 | 28 | func TestStorageSharedLB(t *testing.T) { 29 | key := types.NamespacedName{ 30 | Name: "foo", 31 | Namespace: "default", 32 | } 33 | created := &SharedLB{ 34 | ObjectMeta: metav1.ObjectMeta{ 35 | Name: "foo", 36 | Namespace: "default", 37 | }} 38 | g := gomega.NewGomegaWithT(t) 39 | 40 | // Test Create 41 | fetched := &SharedLB{} 42 | g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) 43 | 44 | g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) 45 | g.Expect(fetched).To(gomega.Equal(created)) 46 | 47 | // Test Updating the Labels 48 | updated := fetched.DeepCopy() 49 | updated.Labels = map[string]string{"hello": "world"} 50 | g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) 51 | 52 | g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) 53 | g.Expect(fetched).To(gomega.Equal(updated)) 54 | 55 | // Test Delete 56 | g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) 57 | g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) 58 | } 59 | -------------------------------------------------------------------------------- /pkg/providers/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "math/rand" 21 | "os" 22 | "strconv" 23 | "time" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/types" 27 | ) 28 | 29 | func init() { 30 | rand.Seed(time.Now().UnixNano()) 31 | } 32 | 33 | var letterRunes = []rune("0123456789abcdefghijklmnopqrstuvwxyz") 34 | 35 | func RandStringRunes(n int) string { 36 | b := make([]rune, n) 37 | for i := range b { 38 | b[i] = letterRunes[rand.Intn(len(letterRunes))] 39 | } 40 | return string(b) 41 | } 42 | 43 | func GetEnvVal(envKey, defaultVal string) string { 44 | if val := os.Getenv(envKey); val != "" { 45 | return val 46 | } 47 | return defaultVal 48 | } 49 | 50 | func GetEnvValInt(envKey string, defaultVal int) int { 51 | val := os.Getenv(envKey) 52 | if val == "" { 53 | return defaultVal 54 | } 55 | retVal, err := strconv.Atoi(val) 56 | if err != nil { 57 | return defaultVal 58 | } 59 | return retVal 60 | } 61 | 62 | func GetNamespacedName(svc *corev1.Service) types.NamespacedName { 63 | if svc == nil { 64 | return types.NamespacedName{} 65 | } 66 | return types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace} 67 | } 68 | 69 | // GetRandomInt returns an integer in range [min, max) 70 | func GetRandomInt(min, max int) int { 71 | return rand.Intn(max-min) + min 72 | } 73 | 74 | func GetRandomPort() int32 { 75 | // TODO(Huang-Wei): change to [1000, 65535)? 76 | return int32(GetRandomInt(1000, 10000)) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/apis/kubecon/v1alpha1/sharedlb_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // SharedLBSpec defines the desired state of SharedLB 25 | type SharedLBSpec struct { 26 | Ports []corev1.ServicePort `json:"ports,omitempty"` 27 | Selector map[string]string `json:"selector,omitempty"` 28 | LoadBalancerIP string `json:"loadBalancerIP,omitempty"` 29 | } 30 | 31 | // SharedLBStatus defines the observed state of SharedLB 32 | type SharedLBStatus struct { 33 | Ref string `json:"ref,omitempty"` 34 | LoadBalancer corev1.LoadBalancerStatus `json:"loadBalancer,omitempty"` 35 | } 36 | 37 | // +genclient 38 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 39 | 40 | // SharedLB is the Schema for the sharedlbs API 41 | // +k8s:openapi-gen=true 42 | type SharedLB struct { 43 | metav1.TypeMeta `json:",inline"` 44 | metav1.ObjectMeta `json:"metadata,omitempty"` 45 | 46 | Spec SharedLBSpec `json:"spec,omitempty"` 47 | Status SharedLBStatus `json:"status,omitempty"` 48 | } 49 | 50 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 51 | 52 | // SharedLBList contains a list of SharedLB 53 | type SharedLBList struct { 54 | metav1.TypeMeta `json:",inline"` 55 | metav1.ListMeta `json:"metadata,omitempty"` 56 | Items []SharedLB `json:"items"` 57 | } 58 | 59 | func init() { 60 | SchemeBuilder.Register(&SharedLB{}, &SharedLBList{}) 61 | } 62 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | controller-tools.k8s.io: "1.0" 6 | name: system 7 | --- 8 | apiVersion: v1 9 | kind: Service 10 | metadata: 11 | name: controller-manager-service 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | controller-tools.k8s.io: "1.0" 16 | spec: 17 | selector: 18 | control-plane: controller-manager 19 | controller-tools.k8s.io: "1.0" 20 | ports: 21 | - port: 443 22 | --- 23 | apiVersion: apps/v1 24 | kind: StatefulSet 25 | metadata: 26 | name: controller-manager 27 | namespace: system 28 | labels: 29 | control-plane: controller-manager 30 | controller-tools.k8s.io: "1.0" 31 | spec: 32 | selector: 33 | matchLabels: 34 | control-plane: controller-manager 35 | controller-tools.k8s.io: "1.0" 36 | serviceName: controller-manager-service 37 | template: 38 | metadata: 39 | labels: 40 | control-plane: controller-manager 41 | controller-tools.k8s.io: "1.0" 42 | spec: 43 | containers: 44 | - command: 45 | - /root/manager 46 | image: controller:latest 47 | imagePullPolicy: Always 48 | name: manager 49 | env: 50 | - name: POD_NAMESPACE 51 | valueFrom: 52 | fieldRef: 53 | fieldPath: metadata.namespace 54 | - name: SECRET_NAME 55 | value: $(WEBHOOK_SECRET_NAME) 56 | resources: 57 | limits: 58 | cpu: 100m 59 | memory: 30Mi 60 | requests: 61 | cpu: 100m 62 | memory: 20Mi 63 | ports: 64 | - containerPort: 9876 65 | name: webhook-server 66 | protocol: TCP 67 | volumeMounts: 68 | - mountPath: /tmp/cert 69 | name: cert 70 | readOnly: true 71 | terminationGracePeriodSeconds: 10 72 | volumes: 73 | - name: cert 74 | secret: 75 | defaultMode: 420 76 | secretName: webhook-server-secret 77 | --- 78 | apiVersion: v1 79 | kind: Secret 80 | metadata: 81 | name: webhook-server-secret 82 | namespace: system 83 | -------------------------------------------------------------------------------- /demos/eks/demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source $(dirname "$0")/../demoscript 4 | source ~/.eks 5 | 6 | comment "This is a demo on Elastic (Amazon) Kubernetes Service" 7 | 8 | doit source ~/.eks 9 | doit echo "\$KUBECONFIG" 10 | 11 | doit kubectl version --short 12 | 13 | doit kubectl get svc 14 | 15 | comment "We are going to create some TCP deployments" 16 | 17 | doit ls demos/eks/deployments 18 | doit kubectl create -f demos/eks/deployments/ 19 | 20 | comment Custom Resource Definition has been pre-created 21 | doit kubectl get crd 22 | doit kubectl get slb 23 | 24 | comment To demo the case LB resource is out of capacity, CAPACITY is configured to 2 25 | doit cat demos/eks/crs/cr-tcp1.yaml 26 | doit kubectl create -f demos/eks/crs/cr-tcp1.yaml 27 | # doit kubectl get svc 28 | 29 | comment EKS LB needs ~3 seconds to be created 30 | doit kubectl get svc 31 | doit kubectl get slb 32 | doit kubectl get slb -o custom-columns="NAME:metadata.name,EXTERNAL-IP:.status.loadBalancer.ingress[*].*,PORT:.spec.ports[*].port" 33 | 34 | comment "Let's take a look at AWS console" 35 | 36 | doit kubectl create -f demos/eks/crs/cr-tcp2.yaml 37 | doit kubectl get svc 38 | doit kubectl get slb -o custom-columns="NAME:metadata.name,EXTERNAL-IP:.status.loadBalancer.ingress[*].*,PORT:.spec.ports[*].port" 39 | 40 | comment "As of now, we're out of capacity, so next CR creation is expected to trigger a new LB creation" 41 | 42 | doit kubectl create -f demos/eks/crs/cr-tcp3.yaml 43 | comment Wait for another 3 seconds 44 | doit kubectl get svc 45 | doit kubectl get slb -o custom-columns="NAME:metadata.name,EXTERNAL-IP:.status.loadBalancer.ingress[*].*,PORT:.spec.ports[*].port" 46 | 47 | comment "Finally let's check connectivity" 48 | out=$(kubectl get slb -o custom-columns="NAME:metadata.name,EXTERNAL-IP:.status.loadBalancer.ingress[*].*,PORT:.spec.ports[*].port") 49 | doit nslookup $(echo "$out" | grep sharedlb-tcp1 | awk '{print $2}') 50 | doit nc -zv $(echo "$out" | grep sharedlb-tcp1 | awk '{print $2}') $(echo "$out" | grep sharedlb-tcp1 | awk '{print $3}') 51 | 52 | comment "Let's delete the 1st SharedLB CR" 53 | doit kubectl delete slb sharedlb-tcp1 54 | doit kubectl get svc 55 | doit kubectl get slb 56 | comment "Checkout AWS console again" 57 | 58 | # comment Cleanup 59 | 60 | # doit kubectl delete deploy --all 61 | # doit kubectl delete slb --all 62 | # doit kubectl delete svc -l=lb-template= 63 | 64 | comment ~End~ -------------------------------------------------------------------------------- /pkg/controller/sharedlb/sharedlb_controller_suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sharedlb 18 | 19 | import ( 20 | lg "log" 21 | "os" 22 | "path/filepath" 23 | "sync" 24 | "testing" 25 | 26 | "github.com/Huang-Wei/shared-loadbalancer/pkg/apis" 27 | "github.com/onsi/gomega" 28 | "k8s.io/client-go/kubernetes/scheme" 29 | "k8s.io/client-go/rest" 30 | "sigs.k8s.io/controller-runtime/pkg/envtest" 31 | "sigs.k8s.io/controller-runtime/pkg/manager" 32 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 33 | ) 34 | 35 | var cfg *rest.Config 36 | 37 | func TestMain(m *testing.M) { 38 | t := &envtest.Environment{ 39 | CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, 40 | } 41 | apis.AddToScheme(scheme.Scheme) 42 | 43 | var err error 44 | if cfg, err = t.Start(); err != nil { 45 | lg.Fatal(err) 46 | } 47 | 48 | code := m.Run() 49 | t.Stop() 50 | os.Exit(code) 51 | } 52 | 53 | // SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and 54 | // writes the request to requests after Reconcile is finished. 55 | func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { 56 | requests := make(chan reconcile.Request) 57 | fn := reconcile.Func(func(req reconcile.Request) (reconcile.Result, error) { 58 | result, err := inner.Reconcile(req) 59 | requests <- req 60 | return result, err 61 | }) 62 | return fn, requests 63 | } 64 | 65 | // StartTestManager adds recFn 66 | func StartTestManager(mgr manager.Manager, g *gomega.GomegaWithT) (chan struct{}, *sync.WaitGroup) { 67 | stop := make(chan struct{}) 68 | wg := &sync.WaitGroup{} 69 | go func() { 70 | wg.Add(1) 71 | g.Expect(mgr.Start(stop)).NotTo(gomega.HaveOccurred()) 72 | wg.Done() 73 | }() 74 | return stop, wg 75 | } 76 | -------------------------------------------------------------------------------- /cmd/manager/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "os" 21 | 22 | "github.com/Huang-Wei/shared-loadbalancer/pkg/apis" 23 | "github.com/Huang-Wei/shared-loadbalancer/pkg/controller" 24 | "github.com/Huang-Wei/shared-loadbalancer/pkg/webhook" 25 | 26 | // _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 27 | _ "k8s.io/client-go/plugin/pkg/client/auth" 28 | "sigs.k8s.io/controller-runtime/pkg/client/config" 29 | "sigs.k8s.io/controller-runtime/pkg/manager" 30 | logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" 31 | "sigs.k8s.io/controller-runtime/pkg/runtime/signals" 32 | ) 33 | 34 | func main() { 35 | logf.SetLogger(logf.ZapLogger(true)) 36 | log := logf.Log.WithName("entrypoint") 37 | 38 | // Get a config to talk to the apiserver 39 | log.Info("setting up client for manager") 40 | cfg, err := config.GetConfig() 41 | if err != nil { 42 | log.Error(err, "unable to set up client config") 43 | os.Exit(1) 44 | } 45 | 46 | // Create a new Cmd to provide shared dependencies and start components 47 | log.Info("setting up manager") 48 | mgr, err := manager.New(cfg, manager.Options{}) 49 | if err != nil { 50 | log.Error(err, "unable to set up overall controller manager") 51 | os.Exit(1) 52 | } 53 | 54 | log.Info("Registering Components.") 55 | 56 | // Setup Scheme for all resources 57 | log.Info("setting up scheme") 58 | if err := apis.AddToScheme(mgr.GetScheme()); err != nil { 59 | log.Error(err, "unable to add APIs to scheme") 60 | os.Exit(1) 61 | } 62 | 63 | // Setup all Controllers 64 | log.Info("Setting up controller") 65 | if err := controller.AddToManager(mgr); err != nil { 66 | log.Error(err, "unable to register controllers to the manager") 67 | os.Exit(1) 68 | } 69 | 70 | log.Info("setting up webhooks") 71 | if err := webhook.AddToManager(mgr); err != nil { 72 | log.Error(err, "unable to register webhooks to the manager") 73 | os.Exit(1) 74 | } 75 | 76 | // Start the Cmd 77 | log.Info("Starting the Cmd.") 78 | if err := mgr.Start(signals.SetupSignalHandler()); err != nil { 79 | log.Error(err, "unable to run the manager") 80 | os.Exit(1) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shared Kubernetes LoadBalancer 2 | 3 | ## Background 4 | 5 | We know that in Kubernetes, there are generally 3 ways to expose workloads publicly: 6 | 7 | - Service (with type `NodePort`) 8 | - Service (with type `LoadBalancer`) 9 | - Ingress 10 | 11 | > kubectl proxy and similar dev/debug solutions are not counted in. 12 | 13 | `NodePort` Service comes almost as early as born of Kubernetes. But due to limitation on ports range (30000~32767), randomness of port, and the need to expose public network of (almost) the whole cluster, `NodePort` Service is usually not considered as a good L4 solution in serious production workloads. 14 | 15 | A viable solution today for L4 apps is `LoadBalancer` service. It's implemented differently in different Kubernetes offerings, by connecting an Kubernetes Service object with a real/virtual IaaS LoadBalancer, so that traffic going through LoadBalancer endpoint can be routed to destination pods properly. 16 | 17 | However, in reality, L7 (e.g. HTTP) workloads are way more widely used than L4 ones. So community comes up with the `Ingress` concept. `Ingress` object defines how incoming request can be routed to internal Service, and under the hood there is an ingress controller (1) dealing with `Ingress` objects, setting up mapping rules by leveraging Nginx/Envoy/etc. and also (2) (normally) exposing via `LoadBalancer` externally. 18 | 19 | > There is a misunderstanding that using Ingress, it's also doable to manage L4 workloads. It's not true. Why Ingress can work is b/c it can differentiate requests by HTTP headers, but for a L4 packet, it's only ip + port. 20 | 21 | ## Motivation 22 | 23 | Ingress introduces a possibility which enables you to expose multiple internal L7 services through **one** public endpoint. But it doesn't work for L4 workloads. 24 | 25 | ![](docs/pics/4-dimensions.png) 26 | 27 | From the above picture, you might wonder where's the missing piece for L4 services? This is exactly the problem we're trying to solve in this project. And following factors are considered: 28 | 29 | - Cost effective 30 | - User friendly 31 | - Reusing existing Kubernetes assets 32 | - Minimum operation efforts 33 | - Consistent with Kubernetes roadmap 34 | 35 | ## How It Works 36 | 37 | We introduce a "SharedLoadBalancer Controller" to customize current Kubernetes behavior. 38 | 39 | Without a "SharedLoadBalancer Controller", it's N Services (of type LoadBalancer) mapped to N LoadBalancer endpoints: 40 | 41 | ![](docs/pics/current-lb.png) 42 | 43 | With a "SharedLoadBalancer Controller", it's N SharedLB CR objects mapped to 1 LoadBalancer endpoint (on different ports): 44 | 45 | ![](docs/pics/shared-lb.png) 46 | 47 | ## More Info 48 | 49 | Want to get more info on this? Join us at KubeCon + CloudNativeCon North America 2018 in Seattle, December 11-13, we will be giving a [session](https://sched.co/GrUd) on this. 50 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Huang-Wei/shared-loadbalancer 2 | 3 | go 1.12 4 | 5 | require ( 6 | cloud.google.com/go v0.30.0 // indirect 7 | github.com/Azure/azure-sdk-for-go v21.3.0+incompatible 8 | github.com/Azure/go-autorest v10.14.0+incompatible 9 | github.com/aws/aws-sdk-go v1.15.61 10 | github.com/davecgh/go-spew v1.1.1 // indirect 11 | github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect 12 | github.com/dimchansky/utfbom v0.0.0-20181204073223-c410c2305b32 // indirect 13 | github.com/ghodss/yaml v1.0.0 // indirect 14 | github.com/go-logr/logr v0.1.0 15 | github.com/go-logr/zapr v0.1.0 // indirect 16 | github.com/gogo/protobuf v1.1.1 // indirect 17 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect 18 | github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 // indirect 19 | github.com/google/btree v1.0.0 // indirect 20 | github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect 21 | github.com/google/uuid v1.0.0 // indirect 22 | github.com/googleapis/gnostic v0.2.0 // indirect 23 | github.com/gophercloud/gophercloud v0.0.0-20181019014921-0719c6b22f30 // indirect 24 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect 25 | github.com/hashicorp/golang-lru v0.5.0 // indirect 26 | github.com/imdario/mergo v0.3.6 // indirect 27 | github.com/json-iterator/go v1.1.5 // indirect 28 | github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a // indirect 29 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 30 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect 31 | github.com/onsi/gomega v1.4.2 32 | github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect 33 | github.com/peterbourgon/diskv v2.0.1+incompatible // indirect 34 | github.com/pkg/errors v0.8.0 // indirect 35 | github.com/pmezard/go-difflib v1.0.0 // indirect 36 | github.com/spf13/pflag v1.0.3 // indirect 37 | github.com/stretchr/testify v1.2.2 // indirect 38 | go.uber.org/atomic v1.3.2 // indirect 39 | go.uber.org/multierr v1.1.0 // indirect 40 | go.uber.org/zap v1.9.1 // indirect 41 | golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e // indirect 42 | golang.org/x/net v0.0.0-20181017193950-04a2e542c03f 43 | golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4 44 | golang.org/x/sys v0.0.0-20181011152604-fa43e7bc11ba // indirect 45 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect 46 | google.golang.org/api v0.0.0-20181203233308-6142e720c068 47 | google.golang.org/appengine v1.2.0 // indirect 48 | gopkg.in/inf.v0 v0.9.1 // indirect 49 | k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 50 | k8s.io/apiextensions-apiserver v0.0.0-20180808065829-408db4a50408 // indirect 51 | k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d 52 | k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 53 | k8s.io/cloud-provider v0.0.0-20181121074215-9b77dc1c3846 54 | k8s.io/klog v0.1.0 // indirect 55 | k8s.io/kube-openapi v0.0.0-20181018171734-e494cc581111 // indirect 56 | sigs.k8s.io/controller-runtime v0.1.4 57 | sigs.k8s.io/testing_frameworks v0.1.0 // indirect 58 | ) 59 | -------------------------------------------------------------------------------- /pkg/providers/base.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "os" 21 | 22 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 23 | "github.com/go-logr/logr" 24 | corev1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/types" 26 | logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" 27 | ) 28 | 29 | var log logr.Logger 30 | 31 | var ( 32 | // SvcPostfix is the stringa appended to cluster service object 33 | SvcPostfix = "-service" 34 | // namespace that LoadBalancer service will be created in 35 | // most probably it's the same value of the namespace that this binary runs in 36 | namespace = GetEnvVal("NAMESPACE", "default") 37 | // capacity is the threshold value a LoadBalancer service can hold 38 | capacity = GetEnvValInt("CAPACITY", 5) 39 | // FinalizerName is the name of finalizer attached to Cluster Service object 40 | FinalizerName = "sharedlb.kubecon.k8s.io/finalizer" 41 | ) 42 | 43 | type nameSet map[types.NamespacedName]struct{} 44 | type int32Set map[int32]struct{} 45 | 46 | func init() { 47 | log = logf.Log.WithName("providers") 48 | } 49 | 50 | func NewProvider() LBProvider { 51 | providerStr := GetEnvVal("PROVIDER", "local") 52 | log.Info("New LBProvider", "provider", providerStr) 53 | var provider LBProvider 54 | switch providerStr { 55 | case "iks": 56 | provider = newIKSProvider() 57 | case "eks": 58 | provider = newEKSProvider() 59 | case "aks": 60 | provider = newAKSProvider() 61 | case "gke": 62 | provider = newGKEProvider() 63 | case "local": 64 | provider = newLocalProvider() 65 | default: 66 | log.Info("Unsupported provider", "provider", providerStr) 67 | os.Exit(1) 68 | } 69 | 70 | return provider 71 | } 72 | 73 | // LBProvider defines methods that a loadbalancer provider should implement 74 | type LBProvider interface { 75 | NewService(sharedLB *kubeconv1alpha1.SharedLB) *corev1.Service 76 | NewLBService() *corev1.Service 77 | GetAvailabelLB(clusterSvc *corev1.Service) *corev1.Service 78 | AssociateLB(cr, lb types.NamespacedName, clusterSvc *corev1.Service) error 79 | DeassociateLB(cr types.NamespacedName, clusterSvc *corev1.Service) error 80 | UpdateCache(key types.NamespacedName, val *corev1.Service) 81 | GetCapacityPerLB() int 82 | UpdateService(svc, lb *corev1.Service) (portUpdated, externalIPUpdated bool) 83 | } 84 | 85 | func updatePort(svc, lb *corev1.Service, occupiedPorts int32Set) bool { 86 | updated := false 87 | // check if svc carries port info or not 88 | for i, svcPort := range svc.Spec.Ports { 89 | if svcPort.Port != 0 { 90 | continue 91 | } 92 | // TODO: if we run out of random ports.. 93 | for { 94 | randomPort := GetRandomPort() 95 | if _, ok := occupiedPorts[randomPort]; ok { 96 | continue 97 | } 98 | svc.Spec.Ports[i].Port = randomPort 99 | updated = true 100 | break 101 | } 102 | } 103 | return updated 104 | } 105 | -------------------------------------------------------------------------------- /pkg/controller/sharedlb/sharedlb_controller_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sharedlb 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 24 | "github.com/onsi/gomega" 25 | "golang.org/x/net/context" 26 | corev1 "k8s.io/api/core/v1" 27 | apierrors "k8s.io/apimachinery/pkg/api/errors" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/apimachinery/pkg/util/intstr" 31 | "sigs.k8s.io/controller-runtime/pkg/client" 32 | "sigs.k8s.io/controller-runtime/pkg/manager" 33 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 34 | ) 35 | 36 | var c client.Client 37 | 38 | var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "default"}} 39 | var svcKey = types.NamespacedName{Name: "foo-service", Namespace: "default"} 40 | 41 | const timeout = time.Second * 5 42 | 43 | func TestReconcile(t *testing.T) { 44 | g := gomega.NewGomegaWithT(t) 45 | instance := &kubeconv1alpha1.SharedLB{ 46 | ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, 47 | Spec: kubeconv1alpha1.SharedLBSpec{ 48 | Ports: []corev1.ServicePort{ 49 | { 50 | Port: 8080, 51 | TargetPort: intstr.IntOrString{IntVal: 80}, 52 | }, 53 | }, 54 | }, 55 | } 56 | 57 | // Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a 58 | // channel when it is finished. 59 | mgr, err := manager.New(cfg, manager.Options{}) 60 | g.Expect(err).NotTo(gomega.HaveOccurred()) 61 | c = mgr.GetClient() 62 | 63 | recFn, requests := SetupTestReconcile(newReconciler(mgr)) 64 | g.Expect(add(mgr, recFn)).NotTo(gomega.HaveOccurred()) 65 | 66 | stopMgr, mgrStopped := StartTestManager(mgr, g) 67 | 68 | defer func() { 69 | close(stopMgr) 70 | mgrStopped.Wait() 71 | }() 72 | 73 | // Create the SharedLB object and expect the Reconcile and Service to be created 74 | err = c.Create(context.TODO(), instance) 75 | // The instance object may not be a valid object because it might be missing some required fields. 76 | // Please modify the instance object by adding required fields and then remove the following if statement. 77 | if apierrors.IsInvalid(err) { 78 | t.Logf("failed to create object, got an invalid object error: %v", err) 79 | return 80 | } 81 | g.Expect(err).NotTo(gomega.HaveOccurred()) 82 | defer c.Delete(context.TODO(), instance) 83 | g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) 84 | 85 | service := &corev1.Service{} 86 | g.Eventually(func() error { return c.Get(context.TODO(), svcKey, service) }, timeout). 87 | Should(gomega.Succeed()) 88 | 89 | // Delete the Service and expect Reconcile to be called for Service deletion 90 | g.Expect(c.Delete(context.TODO(), service)).NotTo(gomega.HaveOccurred()) 91 | g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) 92 | g.Eventually(func() error { return c.Get(context.TODO(), svcKey, service) }, timeout). 93 | Should(gomega.Succeed()) 94 | 95 | // Manually delete Service since GC isn't enabled in the test control plane 96 | g.Expect(c.Delete(context.TODO(), service)).To(gomega.Succeed()) 97 | 98 | } 99 | -------------------------------------------------------------------------------- /pkg/apis/kubecon/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright 2018 The Shared LoadBalancer Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | // Code generated by main. DO NOT EDIT. 19 | 20 | package v1alpha1 21 | 22 | import ( 23 | v1 "k8s.io/api/core/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *SharedLB) DeepCopyInto(out *SharedLB) { 29 | *out = *in 30 | out.TypeMeta = in.TypeMeta 31 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 32 | in.Spec.DeepCopyInto(&out.Spec) 33 | in.Status.DeepCopyInto(&out.Status) 34 | return 35 | } 36 | 37 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharedLB. 38 | func (in *SharedLB) DeepCopy() *SharedLB { 39 | if in == nil { 40 | return nil 41 | } 42 | out := new(SharedLB) 43 | in.DeepCopyInto(out) 44 | return out 45 | } 46 | 47 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 48 | func (in *SharedLB) DeepCopyObject() runtime.Object { 49 | if c := in.DeepCopy(); c != nil { 50 | return c 51 | } 52 | return nil 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *SharedLBList) DeepCopyInto(out *SharedLBList) { 57 | *out = *in 58 | out.TypeMeta = in.TypeMeta 59 | out.ListMeta = in.ListMeta 60 | if in.Items != nil { 61 | in, out := &in.Items, &out.Items 62 | *out = make([]SharedLB, len(*in)) 63 | for i := range *in { 64 | (*in)[i].DeepCopyInto(&(*out)[i]) 65 | } 66 | } 67 | return 68 | } 69 | 70 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharedLBList. 71 | func (in *SharedLBList) DeepCopy() *SharedLBList { 72 | if in == nil { 73 | return nil 74 | } 75 | out := new(SharedLBList) 76 | in.DeepCopyInto(out) 77 | return out 78 | } 79 | 80 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 81 | func (in *SharedLBList) DeepCopyObject() runtime.Object { 82 | if c := in.DeepCopy(); c != nil { 83 | return c 84 | } 85 | return nil 86 | } 87 | 88 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 89 | func (in *SharedLBSpec) DeepCopyInto(out *SharedLBSpec) { 90 | *out = *in 91 | if in.Ports != nil { 92 | in, out := &in.Ports, &out.Ports 93 | *out = make([]v1.ServicePort, len(*in)) 94 | copy(*out, *in) 95 | } 96 | if in.Selector != nil { 97 | in, out := &in.Selector, &out.Selector 98 | *out = make(map[string]string, len(*in)) 99 | for key, val := range *in { 100 | (*out)[key] = val 101 | } 102 | } 103 | return 104 | } 105 | 106 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharedLBSpec. 107 | func (in *SharedLBSpec) DeepCopy() *SharedLBSpec { 108 | if in == nil { 109 | return nil 110 | } 111 | out := new(SharedLBSpec) 112 | in.DeepCopyInto(out) 113 | return out 114 | } 115 | 116 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 117 | func (in *SharedLBStatus) DeepCopyInto(out *SharedLBStatus) { 118 | *out = *in 119 | in.LoadBalancer.DeepCopyInto(&out.LoadBalancer) 120 | return 121 | } 122 | 123 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharedLBStatus. 124 | func (in *SharedLBStatus) DeepCopy() *SharedLBStatus { 125 | if in == nil { 126 | return nil 127 | } 128 | out := new(SharedLBStatus) 129 | in.DeepCopyInto(out) 130 | return out 131 | } 132 | -------------------------------------------------------------------------------- /pkg/providers/local.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | 23 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/types" 27 | ) 28 | 29 | // DEPRECATED: only for testing 30 | type Local struct { 31 | // key is namespacedName of a LB Serivce, val is the service 32 | cacheMap map[types.NamespacedName]*corev1.Service 33 | 34 | // cr to LB is 1:1 mapping 35 | crToLB map[types.NamespacedName]types.NamespacedName 36 | // lb to CRD is 1:N mapping 37 | lbToCRs map[types.NamespacedName]nameSet 38 | // lbToPorts is keyed with ns/name of a LB, and valued with ports info it holds 39 | lbToPorts map[types.NamespacedName]int32Set 40 | 41 | capacityPerLB int 42 | } 43 | 44 | var _ LBProvider = &Local{} 45 | 46 | func newLocalProvider() *Local { 47 | return &Local{ 48 | cacheMap: make(map[types.NamespacedName]*corev1.Service), 49 | crToLB: make(map[types.NamespacedName]types.NamespacedName), 50 | lbToCRs: make(map[types.NamespacedName]nameSet), 51 | lbToPorts: make(map[types.NamespacedName]int32Set), 52 | capacityPerLB: capacity, 53 | } 54 | } 55 | 56 | func (l *Local) GetCapacityPerLB() int { 57 | return l.capacityPerLB 58 | } 59 | 60 | func (l *Local) UpdateCache(key types.NamespacedName, val *corev1.Service) { 61 | l.cacheMap[key] = val 62 | } 63 | 64 | func (l *Local) NewService(sharedLB *kubeconv1alpha1.SharedLB) *corev1.Service { 65 | return &corev1.Service{ 66 | ObjectMeta: metav1.ObjectMeta{ 67 | Name: sharedLB.Name + SvcPostfix, 68 | Namespace: sharedLB.Namespace, 69 | }, 70 | Spec: corev1.ServiceSpec{ 71 | Ports: sharedLB.Spec.Ports, 72 | Selector: sharedLB.Spec.Selector, 73 | }, 74 | } 75 | } 76 | 77 | func (l *Local) NewLBService() *corev1.Service { 78 | return &corev1.Service{ 79 | ObjectMeta: metav1.ObjectMeta{ 80 | Name: "lb-" + RandStringRunes(8), 81 | Namespace: namespace, 82 | Labels: map[string]string{"lb-template": ""}, 83 | }, 84 | Spec: corev1.ServiceSpec{ 85 | Ports: []corev1.ServicePort{ 86 | { 87 | Port: 33333, 88 | }, 89 | }, 90 | Type: corev1.ServiceTypeLoadBalancer, 91 | }, 92 | } 93 | } 94 | 95 | func (l *Local) GetAvailabelLB(clusterSvc *corev1.Service) *corev1.Service { 96 | // we leverage the randomness of golang "for range" when iterating 97 | OUTERLOOP: 98 | for lbKey, lbSvc := range l.cacheMap { 99 | if len(l.lbToCRs[lbKey]) >= l.capacityPerLB { 100 | continue 101 | } 102 | // must satisfy that all svc ports are not occupied in lbSvc 103 | for _, svcPort := range clusterSvc.Spec.Ports { 104 | if l.lbToPorts[lbKey] == nil { 105 | l.lbToPorts[lbKey] = int32Set{} 106 | } 107 | if _, ok := l.lbToPorts[lbKey][svcPort.Port]; ok { 108 | log.WithName("local").Info(fmt.Sprintf("incoming service has port conflict with lbSvc %q on port %d", lbKey, svcPort.Port)) 109 | continue OUTERLOOP 110 | } 111 | } 112 | return lbSvc 113 | } 114 | return nil 115 | } 116 | 117 | func (l *Local) AssociateLB(crName, lbName types.NamespacedName, clusterSvc *corev1.Service) error { 118 | if clusterSvc != nil { 119 | // for a local (dev) env, we need some 3rd party network L2/L3 solution to work 120 | // as a "LoadBalancer" provider, e.g. metallb 121 | if lbSvc, ok := l.cacheMap[lbName]; !ok || len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 122 | return errors.New("LoadBalancer service not exist yet") 123 | } 124 | // upon program starts, l.lbToPorts[lbName] can be nil 125 | if l.lbToPorts[lbName] == nil { 126 | l.lbToPorts[lbName] = int32Set{} 127 | } 128 | // update crToPorts 129 | for _, svcPort := range clusterSvc.Spec.Ports { 130 | l.lbToPorts[lbName][svcPort.Port] = struct{}{} 131 | } 132 | } 133 | 134 | // following code might be called multiple times, but shouldn't impact 135 | // performance a lot as all of them are O(1) operation 136 | _, ok := l.lbToCRs[lbName] 137 | if !ok { 138 | l.lbToCRs[lbName] = make(nameSet) 139 | } 140 | l.lbToCRs[lbName][crName] = struct{}{} 141 | l.crToLB[crName] = lbName 142 | log.WithName("local").Info("AssociateLB", "cr", crName, "lb", lbName) 143 | return nil 144 | } 145 | 146 | func (l *Local) DeassociateLB(crName types.NamespacedName, clusterSvc *corev1.Service) error { 147 | // update internal cache 148 | if lb, ok := l.crToLB[crName]; ok { 149 | delete(l.crToLB, crName) 150 | delete(l.lbToCRs[lb], crName) 151 | for _, svcPort := range clusterSvc.Spec.Ports { 152 | delete(l.lbToPorts[lb], svcPort.Port) 153 | } 154 | log.WithName("local").Info("DeassociateLB", "crName", crName, "lb", lb) 155 | } 156 | return nil 157 | } 158 | 159 | func (l *Local) UpdateService(svc, lb *corev1.Service) (bool, bool) { 160 | // nothing to do with local provider here 161 | return false, false 162 | } 163 | -------------------------------------------------------------------------------- /pkg/providers/aks_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the Licensa. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apacha.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the Licensa. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | 23 | "github.com/Azure/go-autorest/autorest/to" 24 | 25 | "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" 26 | ) 27 | 28 | func TestUnionLBRules(t *testing.T) { 29 | tests := []struct { 30 | name string 31 | existingRules []network.LoadBalancingRule 32 | expectedRules []network.LoadBalancingRule 33 | want []network.LoadBalancingRule 34 | }{ 35 | { 36 | name: "union simple case", 37 | existingRules: []network.LoadBalancingRule{ 38 | { 39 | Name: to.StringPtr("rule1"), 40 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 41 | FrontendPort: to.Int32Ptr(81), 42 | }, 43 | }, 44 | { 45 | Name: to.StringPtr("rule2"), 46 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 47 | FrontendPort: to.Int32Ptr(82), 48 | }, 49 | }, 50 | }, 51 | expectedRules: []network.LoadBalancingRule{ 52 | { 53 | Name: to.StringPtr("rule3"), 54 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 55 | FrontendPort: to.Int32Ptr(83), 56 | }, 57 | }, 58 | { 59 | Name: to.StringPtr("rule4"), 60 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 61 | FrontendPort: to.Int32Ptr(84), 62 | }, 63 | }, 64 | }, 65 | want: []network.LoadBalancingRule{ 66 | { 67 | Name: to.StringPtr("rule1"), 68 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 69 | FrontendPort: to.Int32Ptr(81), 70 | }, 71 | }, 72 | { 73 | Name: to.StringPtr("rule2"), 74 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 75 | FrontendPort: to.Int32Ptr(82), 76 | }, 77 | }, 78 | { 79 | Name: to.StringPtr("rule3"), 80 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 81 | FrontendPort: to.Int32Ptr(83), 82 | }, 83 | }, 84 | { 85 | Name: to.StringPtr("rule4"), 86 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 87 | FrontendPort: to.Int32Ptr(84), 88 | }, 89 | }, 90 | }, 91 | }, 92 | } 93 | for _, tt := range tests { 94 | t.Run(tt.name, func(t *testing.T) { 95 | if _, got := unionLBRules(tt.existingRules, tt.expectedRules); !reflect.DeepEqual(got, tt.want) { 96 | t.Errorf("unionLBRules() = %v, want %v", got, tt.want) 97 | } 98 | }) 99 | } 100 | } 101 | 102 | func TestSubtractLBRules(t *testing.T) { 103 | tests := []struct { 104 | name string 105 | existingRules []network.LoadBalancingRule 106 | unexpectedRules []network.LoadBalancingRule 107 | want []network.LoadBalancingRule 108 | }{ 109 | { 110 | name: "subtract simple case", 111 | existingRules: []network.LoadBalancingRule{ 112 | { 113 | Name: to.StringPtr("rule1"), 114 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 115 | FrontendPort: to.Int32Ptr(81), 116 | }, 117 | }, 118 | { 119 | Name: to.StringPtr("rule2"), 120 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 121 | FrontendPort: to.Int32Ptr(82), 122 | }, 123 | }, 124 | { 125 | Name: to.StringPtr("rule3"), 126 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 127 | FrontendPort: to.Int32Ptr(83), 128 | }, 129 | }, 130 | { 131 | Name: to.StringPtr("rule4"), 132 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 133 | FrontendPort: to.Int32Ptr(84), 134 | }, 135 | }, 136 | }, 137 | unexpectedRules: []network.LoadBalancingRule{ 138 | { 139 | Name: to.StringPtr("rule1"), 140 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 141 | FrontendPort: to.Int32Ptr(81), 142 | }, 143 | }, 144 | { 145 | Name: to.StringPtr("rule3"), 146 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 147 | FrontendPort: to.Int32Ptr(83), 148 | }, 149 | }, 150 | }, 151 | want: []network.LoadBalancingRule{ 152 | { 153 | Name: to.StringPtr("rule2"), 154 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 155 | FrontendPort: to.Int32Ptr(82), 156 | }, 157 | }, 158 | { 159 | Name: to.StringPtr("rule4"), 160 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 161 | FrontendPort: to.Int32Ptr(84), 162 | }, 163 | }, 164 | }, 165 | }, 166 | } 167 | for _, tt := range tests { 168 | t.Run(tt.name, func(t *testing.T) { 169 | if _, got := subtractLBRules(tt.existingRules, tt.unexpectedRules); !reflect.DeepEqual(got, tt.want) { 170 | t.Errorf("subtractLBRules() = %v, want %v", got, tt.want) 171 | } 172 | }) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /pkg/providers/iks.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | 23 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/types" 27 | ) 28 | 29 | // IKS stands for IBM Kubernetes Service 30 | type IKS struct { 31 | // key is namespacedName of a LB Serivce, val is the service 32 | cacheMap map[types.NamespacedName]*corev1.Service 33 | 34 | // cr to LB is 1:1 mapping 35 | crToLB map[types.NamespacedName]types.NamespacedName 36 | // lb to CRD is 1:N mapping 37 | lbToCRs map[types.NamespacedName]nameSet 38 | // lbToPorts is keyed with ns/name of a LB, and valued with ports info it holds 39 | lbToPorts map[types.NamespacedName]int32Set 40 | 41 | capacityPerLB int 42 | } 43 | 44 | var _ LBProvider = &IKS{} 45 | 46 | func newIKSProvider() *IKS { 47 | return &IKS{ 48 | cacheMap: make(map[types.NamespacedName]*corev1.Service), 49 | crToLB: make(map[types.NamespacedName]types.NamespacedName), 50 | lbToCRs: make(map[types.NamespacedName]nameSet), 51 | lbToPorts: make(map[types.NamespacedName]int32Set), 52 | capacityPerLB: capacity, 53 | } 54 | } 55 | 56 | func (i *IKS) GetCapacityPerLB() int { 57 | return i.capacityPerLB 58 | } 59 | 60 | func (i *IKS) UpdateCache(key types.NamespacedName, lbSvc *corev1.Service) { 61 | if lbSvc == nil { 62 | delete(i.cacheMap, key) 63 | return 64 | } 65 | if len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 66 | return 67 | } 68 | 69 | i.cacheMap[key] = lbSvc 70 | } 71 | 72 | func (i *IKS) NewService(sharedLB *kubeconv1alpha1.SharedLB) *corev1.Service { 73 | return &corev1.Service{ 74 | ObjectMeta: metav1.ObjectMeta{ 75 | Name: sharedLB.Name + SvcPostfix, 76 | Namespace: sharedLB.Namespace, 77 | }, 78 | Spec: corev1.ServiceSpec{ 79 | Ports: sharedLB.Spec.Ports, 80 | Selector: sharedLB.Spec.Selector, 81 | }, 82 | } 83 | } 84 | 85 | func (i *IKS) NewLBService() *corev1.Service { 86 | return &corev1.Service{ 87 | ObjectMeta: metav1.ObjectMeta{ 88 | Name: "lb-" + RandStringRunes(8), 89 | Namespace: namespace, 90 | Labels: map[string]string{"lb-template": ""}, 91 | }, 92 | Spec: corev1.ServiceSpec{ 93 | Ports: []corev1.ServicePort{ 94 | { 95 | Name: "tcp", 96 | Protocol: corev1.ProtocolTCP, 97 | Port: 33333, 98 | }, 99 | // TODO(Huang-Wei): handle UDP case 100 | // { 101 | // Name: "UDP", 102 | // Protocol: corev1.ProtocolUDP, 103 | // Port: 33333, 104 | // }, 105 | }, 106 | Type: corev1.ServiceTypeLoadBalancer, 107 | }, 108 | } 109 | } 110 | 111 | func (i *IKS) GetAvailabelLB(clusterSvc *corev1.Service) *corev1.Service { 112 | // we leverage the randomness of golang "for range" when iterating 113 | OUTERLOOP: 114 | for lbKey, lbSvc := range i.cacheMap { 115 | if len(i.lbToCRs[lbKey]) >= i.capacityPerLB || len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 116 | continue 117 | } 118 | // must satisfy that all svc ports are not occupied in lbSvc 119 | for _, svcPort := range clusterSvc.Spec.Ports { 120 | if i.lbToPorts[lbKey] == nil { 121 | i.lbToPorts[lbKey] = int32Set{} 122 | } 123 | if _, ok := i.lbToPorts[lbKey][svcPort.Port]; ok { 124 | log.WithName("iks").Info(fmt.Sprintf("incoming service has port conflict with lbSvc %q on port %d", lbKey, svcPort.Port)) 125 | continue OUTERLOOP 126 | } 127 | } 128 | return lbSvc 129 | } 130 | return nil 131 | } 132 | 133 | func (i *IKS) AssociateLB(crName, lbName types.NamespacedName, clusterSvc *corev1.Service) error { 134 | if clusterSvc != nil { 135 | if lbSvc, ok := i.cacheMap[lbName]; !ok || len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 136 | return errors.New("LoadBalancer service not exist yet") 137 | } 138 | // upon program starts, i.lbToPorts[lbName] can be nil 139 | if i.lbToPorts[lbName] == nil { 140 | i.lbToPorts[lbName] = int32Set{} 141 | } 142 | // update crToPorts 143 | for _, svcPort := range clusterSvc.Spec.Ports { 144 | i.lbToPorts[lbName][svcPort.Port] = struct{}{} 145 | } 146 | } 147 | 148 | // following code might be called multiple times, but shouldn't impact 149 | // performance a lot as all of them are O(1) operation 150 | _, ok := i.lbToCRs[lbName] 151 | if !ok { 152 | i.lbToCRs[lbName] = make(nameSet) 153 | } 154 | i.lbToCRs[lbName][crName] = struct{}{} 155 | i.crToLB[crName] = lbName 156 | log.WithName("iks").Info("AssociateLB", "cr", crName, "lb", lbName) 157 | return nil 158 | } 159 | 160 | // DeassociateLB is called by IKS finalizer to clean internal cache 161 | // no IaaS things should be done for IKS 162 | func (i *IKS) DeassociateLB(crName types.NamespacedName, clusterSvc *corev1.Service) error { 163 | // update internal cache 164 | if lb, ok := i.crToLB[crName]; ok { 165 | delete(i.crToLB, crName) 166 | delete(i.lbToCRs[lb], crName) 167 | for _, svcPort := range clusterSvc.Spec.Ports { 168 | delete(i.lbToPorts[lb], svcPort.Port) 169 | } 170 | log.WithName("iks").Info("DeassociateLB", "cr", crName, "lb", lb) 171 | } 172 | return nil 173 | } 174 | 175 | func (i *IKS) UpdateService(svc, lb *corev1.Service) (bool, bool) { 176 | lbName := types.NamespacedName{Name: lb.Name, Namespace: lb.Namespace} 177 | occupiedPorts := i.lbToPorts[lbName] 178 | if len(occupiedPorts) == 0 { 179 | occupiedPorts = int32Set{} 180 | } 181 | portUpdated := updatePort(svc, lb, occupiedPorts) 182 | externalIPUpdated := updateExternalIP(svc, lb) 183 | return portUpdated, externalIPUpdated 184 | } 185 | 186 | func updateExternalIP(svc, lb *corev1.Service) bool { 187 | if len(lb.Status.LoadBalancer.Ingress) != 1 { 188 | log.Info("No ingress info in lb.Status.LoadBalancer. Skip.") 189 | return false 190 | } 191 | // for IKS, we're setting loadbalancer info as "externalIP" to the service 192 | ingress := lb.Status.LoadBalancer.Ingress[0] 193 | svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ingress.IP) 194 | log.Info("Setting ExternalIP to service", "externalIP", ingress.IP) 195 | return true 196 | } 197 | -------------------------------------------------------------------------------- /demos/demoscript: -------------------------------------------------------------------------------- 1 | # check out https://github.com/duglin/tools/blob/master/demoscript/demoscript 2 | 3 | # SAVE=1 Save output to a tar file for off-line running 4 | # SKIP=1 Do not wait for user to press a key to continue 5 | # RECOVER=1 Use the canned output when a command fails 6 | # USESAVED=1 Use canned output instead of running the commands 7 | # USESAVED=2 Use canned output IFF it exists, otherwise run it 8 | 9 | scriptName="${0##.*/}" 10 | bold=$(tput bold) 11 | normal=$(tput sgr0) 12 | delay=${DELAY:-"0.02"} 13 | skip="$SKIP" 14 | save="$SAVE" 15 | saveTar=$(cd $(dirname "$0");pwd)/${scriptName}.tar 16 | recover="$RECOVER" 17 | useSaved="$USESAVED" 18 | 19 | trap clean EXIT 20 | 21 | function clean { 22 | rm -f out 23 | } 24 | 25 | if [[ "${useSaved}" != "" && "${useSaved}" != "2" && ! -e "${saveTar}" ]]; then 26 | echo "Missing saved output file: ${saveTar}" 27 | exit 1 28 | fi 29 | 30 | if [[ "${save}" != "" && "${useSaved}" == "" ]]; then 31 | rm -f "${saveTar}" 32 | fi 33 | 34 | function myscript() { 35 | if [[ "$(uname)" == Darwin ]]; then 36 | script -q -a /dev/null $* 37 | else 38 | script -efq -a /dev/null -c "$*" 39 | fi 40 | } 41 | 42 | function slowType() { 43 | str="$*" 44 | if [[ "$delay" == "0" ]]; then 45 | echo -n $bold$str$normal 46 | return 47 | fi 48 | for i in `seq 0 ${#str}`; do 49 | echo -n $bold${str:$i:1}$normal 50 | sleep $delay 51 | done 52 | } 53 | 54 | function slowTty() { 55 | str="$*" 56 | echo -n $bold >&3 57 | if [[ "$delay" == "0" ]]; then 58 | echo -n "$str" 59 | else 60 | for i in `seq 0 ${#str}`; do 61 | echo -n "${str:$i:1}" 62 | sleep $delay 63 | done 64 | fi 65 | sleep 0.2 # just to give the other program time to show its input 66 | echo -n $normal >&3 67 | } 68 | 69 | function readChar() { 70 | read -s -n 1 ch 71 | # deal with escape sequences 72 | # ESC[5~ and ESC[6~ are for a mac's page up/down 73 | case "$ch" in 74 | ) read -s -n 1 ch 75 | if [[ "$ch" == "[" ]]; then 76 | read -s -n 1 ch 77 | if [[ "$ch" == "5" || "$ch" == "6" ]]; then 78 | read -s -n 1 ch # should be ~ 79 | fi 80 | fi 81 | ;; 82 | f ) delay="0" ;; 83 | s ) delay=${DELAY:-"0.02"} ;; 84 | r ) skip="x" ;; 85 | esac 86 | } 87 | 88 | function pause() { 89 | if [[ "$skip" == "" ]]; then 90 | readChar 91 | else 92 | sleep 0.2 93 | fi 94 | } 95 | 96 | cmdNum=0 97 | 98 | function doit() { 99 | local ignorerc="" 100 | local shouldfail="" 101 | local noexec="" 102 | local fakeit=${useSaved:-} 103 | local noscroll=${NOSCROLL:-} 104 | local postcmd="" 105 | local retryonfail=${RETRYONFAIL:-} 106 | 107 | while [[ "$1" == "--"* ]]; do 108 | opt="$1" 109 | shift 110 | 111 | case "$opt" in 112 | --ignorerc ) ignorerc="1" ;; 113 | --shouldfail ) shouldfail="1" ;; 114 | --noexec ) noexec="1" ;; 115 | --usesaved ) fakeit="1" ;; 116 | --noscroll ) noscroll="1" ;; 117 | --scroll ) noscroll="" ;; 118 | --retryonfail ) retryonfail="1" ;; 119 | --post* ) postcmd="${opt#*=}" ;; 120 | esac 121 | done 122 | 123 | while true ; do 124 | set +e 125 | echo -n $bold"$"$normal" " 126 | pause 127 | slowType $* 128 | echo "$*" >> cmds 129 | pause 130 | echo 131 | 132 | saveFile="run.${cmdNum}" 133 | local lines=$(tput lines) 134 | let lines=lines-3 135 | moreCMD="more -$lines" 136 | if [[ "$skip" != "" || "$noscroll" != "" ]]; then 137 | moreCMD="cat" 138 | fi 139 | if [[ "$postcmd" != "" ]]; then 140 | moreCMD="$postcmd | $moreCMD" 141 | fi 142 | 143 | # Unless we're told to not execute it, do it 144 | if [[ "$noexec" == "" ]]; then 145 | if [[ "$fakeit" != "" ]]; then 146 | # Faking it! 147 | if tar -xf "${saveTar}" "${saveFile}" > /dev/null 2>&1; then 148 | # echo "** Using saved output ${saveFile} **" 149 | cp "${saveFile}" out 150 | rm "${saveFile}" 151 | cat out | eval ${moreCMD[@]} 152 | 153 | if [[ "$shouldfail" == "" ]]; then 154 | rc=0 155 | else 156 | rc=1 157 | fi 158 | else 159 | if [[ "$fakeit" == "2" ]]; then 160 | # file doesn't exist so just try to run it instead 161 | fakeit="" 162 | else 163 | echo -n > out 164 | fi 165 | fi 166 | fi 167 | 168 | if [[ "$fakeit" == "" ]]; then 169 | # Run the cmd 170 | bash -c " $* " 2>&1 | tee out | eval ${moreCMD[@]} 171 | rc=${PIPESTATUS[0]} 172 | 173 | # Save the output if we're asked to 174 | if [[ "$save" != "" ]]; then 175 | cp out "${saveFile}" 176 | # tar --delete -f "${saveTar}" "${saveFile}" > /dev/null 2>&1 || true 177 | tar -rf "${saveTar}" "${saveFile}" 178 | rm "${saveFile}" 179 | fi 180 | fi 181 | 182 | # If the cmd failed see if we should use the canned output 183 | if [[ "$recover" != "" ]]; then 184 | if [[ ( ( "$shouldfail" == "" && "$rc" != "0" ) || \ 185 | ( "$shouldfail" != "" && "$rc" == "0" ) ) ]] && \ 186 | tar -xf "${saveTar}" "${saveFile}" > /dev/null 2>&1 ; then 187 | # echo "** Using saved output ${saveFile} **" 188 | cp "${saveFile}" out 189 | rm "${saveFile}" 190 | fi 191 | fi 192 | let cmdNum=cmdNum+1 193 | else 194 | # We're not really executing it, just showing the cmd 195 | echo -n > out 196 | rc=0 197 | fi 198 | 199 | echo 200 | 201 | errorMsg="" 202 | 203 | if [[ "$ignorerc" == "" ]]; then 204 | # We're not totally ignoring the exit code 205 | if [[ "$shouldfail" != "" ]]; then 206 | # We need to make sure the command failed as expected 207 | if [[ "$rc" == "0" ]]; then 208 | errorMsg="Expected non-zero exit code, got: $rc" 209 | fi 210 | else 211 | # Normal non-zero exit code expected case 212 | if [[ "$rc" != "0" ]]; then 213 | errorMsg="Non-zero exit code: $rc" 214 | fi 215 | fi 216 | fi 217 | 218 | set -e 219 | 220 | if [[ "${errorMsg}" != "" ]]; then 221 | if [[ "${retryonfail}" != "" ]]; then 222 | echo ${errorMsg}. Retrying 223 | continue 224 | fi 225 | echo ${errorMsg} 226 | exit 1 227 | fi 228 | 229 | break 230 | 231 | done 232 | } 233 | 234 | function background() { 235 | echo -n $bold"$"$normal" " 236 | slowType $* 237 | echo "$*" >> cmds 238 | echo 239 | bash -c " $* " & 240 | } 241 | 242 | function ttyDoit() { 243 | local ignorerc="" 244 | local shouldfail="" 245 | 246 | while [[ "$1" == "--"* ]]; do 247 | opt="$1" 248 | shift 249 | 250 | case "$opt" in 251 | --ignorerc ) ignorerc="1" ;; 252 | --shouldfail ) shouldfail="1" ;; 253 | esac 254 | done 255 | 256 | echo -n $bold"$"$normal" " 257 | pause 258 | slowType "$*" 259 | pause 260 | echo 261 | 262 | exec 3>&1 263 | set +e 264 | ( 265 | sleep 0.2 266 | while read -u 10 line ; do 267 | dontWait="" 268 | if [[ "$line" == "run "* ]]; then 269 | line=${line:4} 270 | ${line} 271 | continue 272 | fi 273 | if [[ "$line" == "@"* ]]; then 274 | # Lines starting with "@" will be executed 275 | # immediately w/o pausing before or after showing it 276 | dontWait="x" 277 | line=${line:1} 278 | fi 279 | if [[ "$dontWait" == "" ]]; then pause ; fi 280 | slowTty $line 281 | if [[ "$dontWait" == "" ]]; then pause ; fi 282 | echo 283 | sleep 0.2 284 | done 285 | echo 286 | ) | myscript $* 287 | rc=${PIPESTATUS[1]} 288 | echo -n $normal 289 | echo 290 | [[ "$ignorerc" == "" && "$rc" != "0" ]] && echo "Non-zero exit code" && exit 1 291 | [[ "$shouldfail" != "" && "$rc" == "0" ]] && echo "Expected non-zero exit code" && exit 1 292 | set -e 293 | } 294 | 295 | function comment() { 296 | local LF="\\n" 297 | local CR=${LF} 298 | local echoopt="" 299 | local dopause="" 300 | local dopauseafter="" 301 | local nohash="" 302 | 303 | while [[ "$1" == "--"* ]]; do 304 | opt="$1" 305 | shift 306 | 307 | case "$opt" in 308 | --nolf ) LF="" ;; 309 | --nocr ) CR="" ; LF="" ;; 310 | --pause ) dopause="1" ; dopauseafter="1" ;; 311 | --pauseafter ) dopauseafter="1" ;; 312 | --nohash ) nohash="1" ;; 313 | esac 314 | done 315 | if [[ "$nohash" == "" ]]; then 316 | echo -en $bold\#" "$normal 317 | fi 318 | if [[ "$dopause" == "1" ]]; then 319 | pause 320 | fi 321 | echo -en ${echoopt} "$bold$*$normal" 322 | if [[ "$dopause" == "1" || "$dopauseafter" == "1" ]]; then 323 | pause 324 | fi 325 | echo -en ${echoopt} "${CR}${LF}" 326 | } 327 | 328 | # Wait until the passed in cmd returns true 329 | function wait() { 330 | # set -x 331 | if [[ "${useSaved}" != "" ]]; then 332 | return 333 | fi 334 | if [ "$1" == "!" ]; then 335 | shift 336 | while (bash -c " $* " &> /dev/null); do 337 | sleep 1 338 | done 339 | else 340 | while !(bash -c " $* " &> /dev/null); do 341 | sleep 1 342 | done 343 | fi 344 | # set +x 345 | } 346 | 347 | function scroll() { 348 | local lines=$(tput lines) 349 | let lines=lines-3 350 | 351 | echo -n $bold"$"$normal" " 352 | # set +e 353 | pause 354 | if [[ "$skip" == "" ]]; then 355 | slowType more $* 356 | else 357 | slowType cat $* 358 | fi 359 | pause 360 | echo 361 | if [[ "$skip" == "" ]]; then 362 | more -$lines $* 363 | else 364 | cat $* 365 | fi 366 | echo 367 | } -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.30.0 h1:xKvyLgk56d0nksWq49J0UyGEeUIicTl4+UBiX1NPX9g= 2 | cloud.google.com/go v0.30.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/Azure/azure-sdk-for-go v21.3.0+incompatible h1:YFvAka2WKAl2xnJkYV1e1b7E2z88AgFszDzWU18ejMY= 4 | github.com/Azure/azure-sdk-for-go v21.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= 5 | github.com/Azure/go-autorest v10.14.0+incompatible h1:oYvXJiRv84OncuWhE2vJUkBxTdvRnvgrn1q1ckmCkzo= 6 | github.com/Azure/go-autorest v10.14.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= 7 | github.com/aws/aws-sdk-go v1.15.61 h1:M1mnQshHau/YfY2hV45rsaAevdMgLp7zh0oHRCgX100= 8 | github.com/aws/aws-sdk-go v1.15.61/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 12 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 13 | github.com/dimchansky/utfbom v0.0.0-20181204073223-c410c2305b32 h1:Fluod+VtCe7vmohCsIuYJ2zhVAvZ+x60/wNZDjetTcA= 14 | github.com/dimchansky/utfbom v0.0.0-20181204073223-c410c2305b32/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= 15 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 16 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 17 | github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= 18 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 19 | github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= 20 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 21 | github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= 22 | github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= 23 | github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= 24 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 25 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 26 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 27 | github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 h1:6UVLWz0fIIrv0UVj6t0A7cL48n8IyAdLVQqAYzEfsKI= 28 | github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 29 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 30 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 31 | github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= 32 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 33 | github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= 34 | github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 35 | github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= 36 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 37 | github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= 38 | github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 39 | github.com/gophercloud/gophercloud v0.0.0-20181019014921-0719c6b22f30 h1:+150LEnnwtkfkSEIaMF/3tRxrq1giMi56N912kCMDHw= 40 | github.com/gophercloud/gophercloud v0.0.0-20181019014921-0719c6b22f30/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= 41 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= 42 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 43 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 44 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 45 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 46 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 47 | github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= 48 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 49 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= 50 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 51 | github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= 52 | github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 53 | github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8= 54 | github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 56 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 57 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 58 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 59 | github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= 60 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 61 | github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= 62 | github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 63 | github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE= 64 | github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= 65 | github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= 66 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 67 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 68 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 69 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 70 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 71 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 72 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 73 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 74 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 75 | go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= 76 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 77 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 78 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 79 | go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= 80 | go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 81 | golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e h1:IzypfodbhbnViNUO/MEh0FzCUooG97cIGfdggUrUSyU= 82 | golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 83 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 84 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 85 | golang.org/x/net v0.0.0-20181017193950-04a2e542c03f h1:4pRM7zYwpBjCnfA1jRmhItLxYJkaEnsmuAcRtA347DA= 86 | golang.org/x/net v0.0.0-20181017193950-04a2e542c03f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 87 | golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4 h1:99CA0JJbUX4ozCnLon680Jc9e0T1i8HCaLVJMwtI8Hc= 88 | golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 89 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= 90 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 91 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 92 | golang.org/x/sys v0.0.0-20181011152604-fa43e7bc11ba h1:nZJIJPGow0Kf9bU9QTc1U6OXbs/7Hu4e+cNv+hxH+Zc= 93 | golang.org/x/sys v0.0.0-20181011152604-fa43e7bc11ba/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 94 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 95 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 96 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= 97 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 98 | google.golang.org/api v0.0.0-20181203233308-6142e720c068 h1:u+OEXWKdCFcxRrVBmdqZz/TY/fWUQ7aaCzwd+zITcJQ= 99 | google.golang.org/api v0.0.0-20181203233308-6142e720c068/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 100 | google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= 101 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 102 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 103 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 104 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 105 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 106 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 107 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 108 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 109 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 110 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 111 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 112 | k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 h1:hROmpFC7JMobXFXMmD7ZKZLhDKvr1IKfFJoYS/45G/8= 113 | k8s.io/api v0.0.0-20180712090710-2d6f90ab1293/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= 114 | k8s.io/apiextensions-apiserver v0.0.0-20180808065829-408db4a50408 h1:GcrrWo5PlDjJ6cSFoxKlIy3xH+IvXa/uYs90NxdbEV4= 115 | k8s.io/apiextensions-apiserver v0.0.0-20180808065829-408db4a50408/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= 116 | k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d h1:MZjlsu9igBoVPZkXpIGoxI6EonqNsXXZU7hhvfQLkd4= 117 | k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= 118 | k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 h1:kQX7jEIMYrWV9XqFN4usRaBLzCu7fd/qsCXxbgf3+9g= 119 | k8s.io/client-go v0.0.0-20180806134042-1f13a808da65/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= 120 | k8s.io/cloud-provider v0.0.0-20181121074215-9b77dc1c3846 h1:UIeuDQRH8zU4QXO3TQ+UuK4Tk9CYp4bfA4EB9FlQaiY= 121 | k8s.io/cloud-provider v0.0.0-20181121074215-9b77dc1c3846/go.mod h1:LlIffnLBu+GG7d4ppPzC8UnA1Ex8S+ntmSRVsnr7Xy4= 122 | k8s.io/klog v0.1.0 h1:I5HMfc/DtuVaGR1KPwUrTc476K8NCqNBldC7H4dYEzk= 123 | k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 124 | k8s.io/kube-openapi v0.0.0-20181018171734-e494cc581111 h1:rVN39diWB9V32OJJ/cK/LrNS6hfYbsHyFx4z6Gi0NzA= 125 | k8s.io/kube-openapi v0.0.0-20181018171734-e494cc581111/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= 126 | sigs.k8s.io/controller-runtime v0.1.4 h1:vzpPotcU0qI9BAEyX8opCzI6NqnINHWk974kJtAeNZo= 127 | sigs.k8s.io/controller-runtime v0.1.4/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= 128 | sigs.k8s.io/testing_frameworks v0.1.0 h1:2hBE1sDhKWALoqvhi2i/mnQOFZVfWtQFtsfH0QBTI0U= 129 | sigs.k8s.io/testing_frameworks v0.1.0/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= 130 | -------------------------------------------------------------------------------- /pkg/providers/gke.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | "strings" 24 | 25 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 26 | "golang.org/x/oauth2/google" 27 | compute "google.golang.org/api/compute/v1" 28 | "google.golang.org/api/googleapi" 29 | corev1 "k8s.io/api/core/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/types" 32 | ) 33 | 34 | // Ref: 35 | // - gcloud auth application-default login 36 | // - gcloud auth application-default print-access-token 37 | 38 | const PORTRANGE = "30000-32767" 39 | 40 | // GKE stands for Google Kubernetes Engine 41 | type GKE struct { 42 | client *compute.Service 43 | project, region string 44 | 45 | // key is namespacedName of a LB Serivce, val is the service 46 | cacheMap map[types.NamespacedName]*corev1.Service 47 | 48 | // cr to LB is 1:1 mapping 49 | crToLB map[types.NamespacedName]types.NamespacedName 50 | // lb to CRD is 1:N mapping 51 | lbToCRs map[types.NamespacedName]nameSet 52 | // lbToPorts is keyed with ns/name of a LB, and valued with ports info it holds 53 | lbToPorts map[types.NamespacedName]int32Set 54 | 55 | capacityPerLB int 56 | } 57 | 58 | var _ LBProvider = &GKE{} 59 | var ctx = context.Background() 60 | 61 | func newGKEProvider() *GKE { 62 | gke := &GKE{ 63 | cacheMap: make(map[types.NamespacedName]*corev1.Service), 64 | crToLB: make(map[types.NamespacedName]types.NamespacedName), 65 | lbToCRs: make(map[types.NamespacedName]nameSet), 66 | lbToPorts: make(map[types.NamespacedName]int32Set), 67 | capacityPerLB: capacity, 68 | } 69 | gke.client = initGCE() 70 | gke.project = GetEnvVal("GKE_PROJ", "kubecon-seattle-project") 71 | gke.region = GetEnvVal("GKE_REGION", "us-central1") 72 | return gke 73 | } 74 | 75 | func initGCE() *compute.Service { 76 | c, err := google.DefaultClient(ctx, compute.CloudPlatformScope) 77 | if err != nil { 78 | log.WithName("gke").Info("Failed to get GCE client") 79 | panic(err) 80 | } 81 | 82 | computeService, err := compute.New(c) 83 | if err != nil { 84 | log.WithName("gke").Info("Failed to create compute service") 85 | panic(err) 86 | } 87 | 88 | return computeService 89 | } 90 | 91 | func (g *GKE) GetCapacityPerLB() int { 92 | return g.capacityPerLB 93 | } 94 | 95 | func (g *GKE) UpdateCache(key types.NamespacedName, lbSvc *corev1.Service) { 96 | if lbSvc == nil { 97 | delete(g.cacheMap, key) 98 | return 99 | } 100 | if len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 101 | return 102 | } 103 | 104 | // Make this IP Static and then add an Inbound Firewall Rule 105 | g.ensureStaticIP(lbSvc.ObjectMeta.Name+"-ip", lbSvc.Status.LoadBalancer.Ingress[0].IP) 106 | // Let us also create an Inbound Firewall Rule to open up all ports. 107 | // We can also do each port the slb needs but for now this is good enough. 108 | g.ensureFirewall(getLBFirewallRuleName(lbSvc.ObjectMeta.Name)) 109 | g.cacheMap[key] = lbSvc 110 | } 111 | 112 | func getLBFirewallRuleName(lbName string) string { 113 | return fmt.Sprintf("k8s-fw-%s-autogen", lbName) 114 | } 115 | 116 | func getLBForwardRuleName(lbName string, port int32, proto corev1.Protocol) string { 117 | // note: proto must be lowered 118 | return fmt.Sprintf("%s-fwd-rule-%d-%v-autogen", lbName, port, strings.ToLower(string(proto))) 119 | } 120 | 121 | func (g *GKE) NewService(sharedLB *kubeconv1alpha1.SharedLB) *corev1.Service { 122 | return &corev1.Service{ 123 | ObjectMeta: metav1.ObjectMeta{ 124 | Name: sharedLB.Name + SvcPostfix, 125 | Namespace: sharedLB.Namespace, 126 | }, 127 | Spec: corev1.ServiceSpec{ 128 | Type: corev1.ServiceTypeNodePort, 129 | Ports: sharedLB.Spec.Ports, 130 | Selector: sharedLB.Spec.Selector, 131 | }, 132 | } 133 | } 134 | 135 | func (g *GKE) NewLBService() *corev1.Service { 136 | return &corev1.Service{ 137 | ObjectMeta: metav1.ObjectMeta{ 138 | Name: "lb-" + RandStringRunes(8), 139 | Namespace: namespace, 140 | Labels: map[string]string{"lb-template": ""}, 141 | }, 142 | Spec: corev1.ServiceSpec{ 143 | Ports: []corev1.ServicePort{ 144 | { 145 | Name: "tcp", 146 | Protocol: corev1.ProtocolTCP, 147 | Port: 33333, 148 | }, 149 | // TODO(Huang-Wei): handle UDP case 150 | // { 151 | // Name: "UDP", 152 | // Protocol: corev1.ProtocolUDP, 153 | // Port: 33333, 154 | // }, 155 | }, 156 | Type: corev1.ServiceTypeLoadBalancer, 157 | }, 158 | } 159 | } 160 | 161 | func (g *GKE) GetAvailabelLB(clusterSvc *corev1.Service) *corev1.Service { 162 | // we leverage the randomness of golang "for range" when iterating 163 | OUTERLOOP: 164 | for lbKey, lbSvc := range g.cacheMap { 165 | if len(g.lbToCRs[lbKey]) >= g.capacityPerLB || len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 166 | continue 167 | } 168 | // must satisfy that all svc ports are not occupied in lbSvc 169 | for _, svcPort := range clusterSvc.Spec.Ports { 170 | if g.lbToPorts[lbKey] == nil { 171 | g.lbToPorts[lbKey] = int32Set{} 172 | } 173 | if _, ok := g.lbToPorts[lbKey][svcPort.Port]; ok { 174 | log.WithName("gke").Info(fmt.Sprintf("incoming service has port conflict with lbSvc %q on port %d", lbKey, svcPort.Port)) 175 | continue OUTERLOOP 176 | } 177 | } 178 | return lbSvc 179 | } 180 | return nil 181 | } 182 | 183 | func (g *GKE) ensureForwardingRule(lbName, ip string, nodePort int32, proto corev1.Protocol) error { 184 | fwdRuleName := getLBForwardRuleName(lbName, nodePort, proto) 185 | if g.getForwardingRule(fwdRuleName) == nil { 186 | lbPool := g.getTargetPool_usingLBName(lbName) 187 | if lbPool == nil { 188 | return errors.New("cannot find the TargetPool for the LoadBalancer") 189 | } 190 | return g.insertForwardingRule(fwdRuleName, ip, nodePort, proto, lbPool.SelfLink) 191 | } 192 | return nil 193 | } 194 | 195 | func (g *GKE) AssociateLB(crName, lbName types.NamespacedName, clusterSvc *corev1.Service) error { 196 | // a) create GCP LoadBalancer Forwarding Rule 197 | if clusterSvc != nil { 198 | if lbSvc := g.cacheMap[lbName]; lbSvc != nil { 199 | for _, svcPort := range clusterSvc.Spec.Ports { 200 | if err := g.ensureForwardingRule(lbName.Name, lbSvc.Status.LoadBalancer.Ingress[0].IP, svcPort.NodePort, svcPort.Protocol); err != nil { 201 | return err 202 | } 203 | } 204 | } 205 | // upon program starts, g.lbToPorts[lbName] can be nil 206 | if g.lbToPorts[lbName] == nil { 207 | g.lbToPorts[lbName] = int32Set{} 208 | } 209 | // update crToPorts 210 | for _, svcPort := range clusterSvc.Spec.Ports { 211 | g.lbToPorts[lbName][svcPort.NodePort] = struct{}{} 212 | } 213 | } 214 | 215 | // c) update internal cache 216 | // following code might be called multiple times, but shouldn't impact 217 | // performance a lot as all of them are O(1) operation 218 | _, ok := g.lbToCRs[lbName] 219 | if !ok { 220 | g.lbToCRs[lbName] = make(nameSet) 221 | } 222 | g.lbToCRs[lbName][crName] = struct{}{} 223 | g.crToLB[crName] = lbName 224 | log.WithName("gke").Info("AssociateLB", "cr", crName, "lb", lbName) 225 | return nil 226 | } 227 | 228 | // DeassociateLB is called by GKE finalizer to clean internal cache 229 | // no IaaS things should be done for GKE 230 | func (g *GKE) DeassociateLB(crName types.NamespacedName, clusterSvc *corev1.Service) error { 231 | lbName, ok := g.crToLB[crName] 232 | if !ok { 233 | return nil 234 | } 235 | 236 | // a) delete GCP LoadBalancer Forwarding rule 237 | if lbSvc := g.cacheMap[lbName]; lbSvc != nil { 238 | for _, svcPort := range clusterSvc.Spec.Ports { 239 | fwdRuleName := getLBForwardRuleName(lbName.Name, svcPort.NodePort, svcPort.Protocol) 240 | if err := g.deleteForwardingRule(fwdRuleName); err != nil { 241 | return err 242 | } 243 | } 244 | } 245 | 246 | // b) update internal cache 247 | delete(g.crToLB, crName) 248 | delete(g.lbToCRs[lbName], crName) 249 | for _, svcPort := range clusterSvc.Spec.Ports { 250 | delete(g.lbToPorts[lbName], svcPort.Port) 251 | } 252 | log.WithName("gke").Info("DeassociateLB", "cr", crName, "lb", lbName) 253 | return nil 254 | } 255 | 256 | func (g *GKE) UpdateService(svc, lb *corev1.Service) (bool, bool) { 257 | lbName := types.NamespacedName{Name: lb.Name, Namespace: lb.Namespace} 258 | occupiedPorts := g.lbToPorts[lbName] 259 | if len(occupiedPorts) == 0 { 260 | occupiedPorts = int32Set{} 261 | } 262 | portUpdated := updatePort(svc, lb, occupiedPorts) 263 | return portUpdated, true 264 | } 265 | 266 | func isAlreadyExist(err error) bool { 267 | apiErr, ok := err.(*googleapi.Error) 268 | return ok && (apiErr.Code == 409 || strings.Contains(apiErr.Message, "alreadyExists")) 269 | } 270 | 271 | func (g *GKE) ensureStaticIP(name, ip string) { 272 | addr := &compute.Address{ 273 | Name: name, 274 | Address: ip, 275 | Region: g.region, 276 | } 277 | 278 | addrInsertCall := g.client.Addresses.Insert(g.project, g.region, addr) 279 | _, err := addrInsertCall.Do() 280 | 281 | if err != nil && !isAlreadyExist(err) { 282 | log.WithName("gke").Error(err, "Faied to secure a Static IP") 283 | } 284 | } 285 | 286 | func (g *GKE) getTargetPool_usingLBName(lb string) *compute.TargetPool { 287 | resp, err := g.client.TargetPools.List(g.project, g.region).Context(ctx).Do() 288 | if err != nil { 289 | log.WithName("gke").Error(err, "Failed to find the loadbalancer "+lb) 290 | } 291 | for _, item := range resp.Items { 292 | // TODO(Huang-Wei): check "name" for exact match 293 | if strings.Contains(item.Description, lb) { 294 | return item 295 | } 296 | } 297 | return nil 298 | } 299 | 300 | func (g *GKE) getForwardingRule(name string) *compute.ForwardingRule { 301 | resp, err := g.client.ForwardingRules.List(g.project, g.region).Context(ctx).Do() 302 | if err != nil { 303 | if strings.Contains(err.Error(), "Error 404") { 304 | return nil 305 | } 306 | log.WithName("gke").Error(err, "Failed to get Forwarding rules") 307 | } 308 | for _, item := range resp.Items { 309 | if strings.Compare(item.Name, name) == 0 { 310 | return item 311 | } 312 | } 313 | return nil 314 | } 315 | 316 | func (g *GKE) insertForwardingRule(name, ip string, port int32, proto corev1.Protocol, item string) error { 317 | fwd := &compute.ForwardingRule{ 318 | IPAddress: ip, 319 | IPProtocol: strings.ToUpper(string(proto)), 320 | Description: "forwarding rule to reach service through nodePort, generated by SharedLB", 321 | LoadBalancingScheme: "EXTERNAL", 322 | Target: item, 323 | Name: name, 324 | PortRange: fmt.Sprintf("%d-%d", port, port), 325 | } 326 | fwdInsertCall := g.client.ForwardingRules.Insert(g.project, g.region, fwd) 327 | _, err := fwdInsertCall.Do() 328 | 329 | if err != nil { 330 | return fmt.Errorf("failed to create the forwarding rule for the item %v: %v", item, err) 331 | } 332 | return nil 333 | } 334 | 335 | func (g *GKE) deleteForwardingRule(name string) error { 336 | fwdDeleteCall := g.client.ForwardingRules.Delete(g.project, g.region, name) 337 | _, err := fwdDeleteCall.Do() 338 | 339 | if err != nil { 340 | return fmt.Errorf("failed to delete the forwarding rule %q", name) 341 | } 342 | return nil 343 | } 344 | 345 | func (g *GKE) ensureFirewall(name string) error { 346 | fw := &compute.Firewall{ 347 | Name: name, 348 | Allowed: []*compute.FirewallAllowed{ 349 | { 350 | IPProtocol: "TCP", 351 | Ports: []string{PORTRANGE}, 352 | }, 353 | { 354 | IPProtocol: "UDP", 355 | Ports: []string{PORTRANGE}, 356 | }, 357 | }, 358 | } 359 | fwInsertCall := g.client.Firewalls.Insert(g.project, fw) 360 | _, err := fwInsertCall.Do() 361 | 362 | if err != nil && !isAlreadyExist(err) { 363 | log.WithName("gke").Error(err, "Unable to add a firewall rule") 364 | } 365 | return err 366 | } 367 | 368 | // not used yet 369 | func (g *GKE) getFirewallRule(name string) *compute.Firewall { 370 | resp, err := g.client.Firewalls.Get(g.project, name).Context(ctx).Do() 371 | if err != nil { 372 | if strings.Contains(err.Error(), "Error 404") { 373 | return nil 374 | } 375 | log.WithName("gke").Error(err, "Failed to get Firewalls rules") 376 | } 377 | return resp 378 | } 379 | 380 | // not used yet 381 | func (g *GKE) firewallDelete(name string) error { 382 | fwDeleteCall := g.client.Firewalls.Delete(g.project, name) 383 | _, err := fwDeleteCall.Do() 384 | 385 | if err != nil { 386 | log.WithName("gke").Error(err, "Failed to delete the firewall rule "+name) 387 | } 388 | return err 389 | } 390 | -------------------------------------------------------------------------------- /pkg/controller/sharedlb/sharedlb_controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sharedlb 18 | 19 | import ( 20 | "context" 21 | "strings" 22 | "time" 23 | 24 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 25 | "github.com/Huang-Wei/shared-loadbalancer/pkg/providers" 26 | "github.com/go-logr/logr" 27 | corev1 "k8s.io/api/core/v1" 28 | "k8s.io/apimachinery/pkg/api/errors" 29 | "k8s.io/apimachinery/pkg/runtime" 30 | "k8s.io/apimachinery/pkg/types" 31 | "sigs.k8s.io/controller-runtime/pkg/client" 32 | "sigs.k8s.io/controller-runtime/pkg/controller" 33 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 34 | "sigs.k8s.io/controller-runtime/pkg/handler" 35 | "sigs.k8s.io/controller-runtime/pkg/manager" 36 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 37 | logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" 38 | "sigs.k8s.io/controller-runtime/pkg/source" 39 | ) 40 | 41 | var log logr.Logger 42 | 43 | func init() { 44 | log = logf.Log.WithName("slb_controller") 45 | } 46 | 47 | // Add creates a new SharedLB Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller 48 | // and Start it when the Manager is Started. 49 | func Add(mgr manager.Manager) error { 50 | return add(mgr, newReconciler(mgr)) 51 | } 52 | 53 | // newReconciler returns a new reconcile.Reconciler 54 | func newReconciler(mgr manager.Manager) reconcile.Reconciler { 55 | return &ReconcileSharedLB{ 56 | Client: mgr.GetClient(), 57 | scheme: mgr.GetScheme(), 58 | provider: providers.NewProvider(), 59 | pendingQ: &pendingQ{ 60 | pendingLB: nil, 61 | pendingCRs: make(map[types.NamespacedName]struct{}), 62 | }, 63 | } 64 | } 65 | 66 | // add adds a new Controller to mgr with r as the reconcile.Reconciler 67 | func add(mgr manager.Manager, r reconcile.Reconciler) error { 68 | // Create a new controller 69 | c, err := controller.New("sharedlb-controller", mgr, controller.Options{Reconciler: r}) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | // Watch LB Service 75 | mapFn := handler.ToRequestsFunc( 76 | func(o handler.MapObject) []reconcile.Request { 77 | _, ok := o.Meta.GetLabels()["lb-template"] 78 | if !ok { 79 | return nil 80 | } 81 | return []reconcile.Request{ 82 | {NamespacedName: types.NamespacedName{ 83 | Name: o.Meta.GetName(), 84 | Namespace: o.Meta.GetNamespace(), 85 | }}, 86 | } 87 | }) 88 | err = c.Watch( 89 | &source.Kind{Type: &corev1.Service{}}, 90 | &handler.EnqueueRequestsFromMapFunc{ 91 | ToRequests: mapFn, 92 | }, 93 | ) 94 | if err != nil { 95 | return err 96 | } 97 | 98 | // Watch for changes to SharedLB 99 | err = c.Watch( 100 | &source.Kind{Type: &kubeconv1alpha1.SharedLB{}}, 101 | &handler.EnqueueRequestForObject{}, 102 | ) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | // Watch a Service created by SharedLB 108 | err = c.Watch( 109 | &source.Kind{Type: &corev1.Service{}}, 110 | &handler.EnqueueRequestForOwner{ 111 | IsController: true, 112 | OwnerType: &kubeconv1alpha1.SharedLB{}, 113 | }, 114 | ) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | return nil 120 | } 121 | 122 | var _ reconcile.Reconciler = &ReconcileSharedLB{} 123 | 124 | // ReconcileSharedLB reconciles a SharedLB object 125 | type ReconcileSharedLB struct { 126 | client.Client 127 | scheme *runtime.Scheme 128 | provider providers.LBProvider 129 | pendingQ *pendingQ 130 | } 131 | 132 | type pendingQ struct { 133 | pendingLB *types.NamespacedName 134 | pendingCRs map[types.NamespacedName]struct{} 135 | } 136 | 137 | // Reconcile reads that state of the cluster for a SharedLB object and makes changes based on the state read 138 | // and what is in the SharedLB.Spec 139 | // The scaffolding writes a Service as an example 140 | // Automatically generate RBAC rules to allow the Controller to read and write Services 141 | // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete 142 | // +kubebuilder:rbac:groups=kubecon.k8s.io,resources=sharedlbs,verbs=get;list;watch;create;update;patch;delete 143 | func (r *ReconcileSharedLB) Reconcile(request reconcile.Request) (reconcile.Result, error) { 144 | // 1) fetch and deal with the LoadBalancer Service object 145 | lbSvc := &corev1.Service{} 146 | err := r.Get(context.TODO(), request.NamespacedName, lbSvc) 147 | if err == nil { 148 | log.Info("LB is created/updated. Updating LB cache.", "name", request.Name) 149 | r.provider.UpdateCache(request.NamespacedName, lbSvc) 150 | if r.pendingQ.hasLB(request.NamespacedName) && len(lbSvc.Status.LoadBalancer.Ingress) > 0 { 151 | log.Info("LB has completed setup. Removing lb entry in pendingQ.") 152 | r.pendingQ.remove(request.NamespacedName) 153 | } 154 | return reconcile.Result{}, nil 155 | } else if errors.IsNotFound(err) && strings.Index(request.Name, "lb-") == 0 { 156 | // TODO(Huang-Wei): improve logic here to check if it's a LB svc deletion 157 | log.Info("LB is deleted. Updating lb cache.", "name", request.Name) 158 | r.provider.UpdateCache(request.NamespacedName, nil) 159 | return reconcile.Result{}, nil 160 | } 161 | 162 | // in some cases, esp. when CR obj is created but LoadBalancer service is pending 163 | // we rely on an internal struct "pendingQ" to tell whether incoming CR obj should 164 | // a) be put back to queue or b) be processed instantly 165 | if r.pendingQ.hasCR(request.NamespacedName) { 166 | log.Info("It's likely a dependent IaaS LoadBalancer is pending. Requeue to wait for its completion and retry.", "request", request.NamespacedName) 167 | // TODO(Huang-Wei): implement exponential backoff 168 | return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 2}, nil 169 | } 170 | 171 | // 2) fetch and deal with the SharedLB CR object 172 | crObj := &kubeconv1alpha1.SharedLB{} 173 | err = r.Get(context.TODO(), request.NamespacedName, crObj) 174 | if err != nil { 175 | if errors.IsNotFound(err) { 176 | // if cr obj is not found, means it's been deleted. we can simply return b/c: 177 | // (1) dependent objects will be automatically garbage collected, and 178 | // (2) additional cleanup logic are handled by finalizers. 179 | return reconcile.Result{}, nil 180 | } 181 | // Error reading the object - requeue the request. 182 | return reconcile.Result{}, err 183 | } 184 | 185 | // add or remove finalizer 186 | if crObj.ObjectMeta.DeletionTimestamp.IsZero() { 187 | // The CR object is not being deleted, so if it does not have desired finalizer, 188 | // add the finalizer and update the object. 189 | if !containsString(crObj.ObjectMeta.Finalizers, providers.FinalizerName) { 190 | crObj.ObjectMeta.Finalizers = append(crObj.ObjectMeta.Finalizers, providers.FinalizerName) 191 | err := r.Update(context.Background(), crObj) 192 | // note: the code here is slightly different with kubebuilder sample code 193 | // where requeue everytime, but here we only requeue when err != nil 194 | return reconcile.Result{Requeue: err != nil}, nil 195 | } 196 | } else { 197 | if containsString(crObj.ObjectMeta.Finalizers, providers.FinalizerName) { 198 | // clusterSvc := r.provider.NewService(crObj) 199 | clusterSvc := &corev1.Service{} 200 | clusterSvcNsName := types.NamespacedName{Name: crObj.Name + providers.SvcPostfix, Namespace: crObj.Namespace} 201 | if err = r.Get(context.TODO(), clusterSvcNsName, clusterSvc); err != nil { 202 | log.Error(err, "fail to get clusterSvc when trying DeassociateLB") 203 | return reconcile.Result{}, err 204 | } 205 | if err := r.provider.DeassociateLB(request.NamespacedName, clusterSvc); err != nil { 206 | // fail to delete external dependencies here, return with error 207 | // so that it can be retried 208 | log.Error(err, "fail to delete external dependencies when trying DeassociateLB") 209 | return reconcile.Result{}, err 210 | } 211 | // remove our finalizer from the list and update it. 212 | crObj.ObjectMeta.Finalizers = removeString(crObj.ObjectMeta.Finalizers, providers.FinalizerName) 213 | err = r.Update(context.Background(), crObj) 214 | // note: the code here is slightly different with kubebuilder sample code 215 | // where requeue everytime, but here we only requeue when err != nil 216 | return reconcile.Result{Requeue: err != nil}, nil 217 | } 218 | } 219 | 220 | // 3) deal with the Cluster Service object 221 | // Define the desired cluster Service object 222 | clusterSvc := r.provider.NewService(crObj) 223 | if err := controllerutil.SetControllerReference(crObj, clusterSvc, r.scheme); err != nil { 224 | return reconcile.Result{}, err 225 | } 226 | 227 | // Check if the cluster Service already exists 228 | found := &corev1.Service{} 229 | err = r.Get(context.TODO(), types.NamespacedName{Name: clusterSvc.Name, Namespace: clusterSvc.Namespace}, found) 230 | 231 | if err == nil && crObj.Status.Ref != "" { 232 | strs := strings.Split(crObj.Status.Ref, "/") 233 | if err := r.provider.AssociateLB(request.NamespacedName, types.NamespacedName{Namespace: strs[0], Name: strs[1]}, clusterSvc); err != nil { 234 | // this err means corresponding IaaS Obj not exist yet 235 | // so we requeue with a bit backoff 236 | // this is possible in 2 cases: 237 | // i) program start phase: CR obj Add event comes before LB obj Add event 238 | // ii) LB/IaaS obj created too slow 239 | log.Info(err.Error()) 240 | return reconcile.Result{Requeue: true, RequeueAfter: time.Millisecond * 100}, nil 241 | } 242 | } 243 | 244 | if err != nil && errors.IsNotFound(err) { 245 | // fetch an available LoadBalancer Service that can be reused 246 | availableLB := r.provider.GetAvailabelLB(clusterSvc) 247 | if availableLB == nil { 248 | if !r.pendingQ.isEmpty() { 249 | r.pendingQ.add(request.NamespacedName, types.NamespacedName{}) 250 | return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 2}, nil 251 | } 252 | log.Info("Creating a real LoadBalancer Service") 253 | availableLB = r.provider.NewLBService() 254 | err = r.Create(context.TODO(), availableLB) 255 | if err != nil { 256 | log.Error(err, "Creating LoadBalancer Service Failed", "name", availableLB.Name) 257 | // backoff a bit 258 | return reconcile.Result{RequeueAfter: time.Second * 1}, err 259 | } 260 | log.Info("A real LB is created", "name", availableLB.Name, "lbinfo", availableLB.Status.LoadBalancer) 261 | // NOTE: here we directly return to start a new reconcile 262 | r.pendingQ.add(request.NamespacedName, types.NamespacedName{Name: availableLB.Name, Namespace: availableLB.Namespace}) 263 | return reconcile.Result{Requeue: true, RequeueAfter: time.Millisecond * 500}, nil 264 | } 265 | 266 | log.Info("Reusing a LoadBalancer Service", "name", availableLB.Name) 267 | lbNamespacedName := types.NamespacedName{Name: availableLB.Name, Namespace: availableLB.Namespace} 268 | // at this point, we can reuse a LoadBalancer 269 | // i.e. availableLB is expected to carry loadbalancer info 270 | // check if this cr carries a port; if not, assign a random port 271 | portUpdated, _ := r.provider.UpdateService(clusterSvc, availableLB) 272 | err = r.Create(context.TODO(), clusterSvc) 273 | if err != nil { 274 | return reconcile.Result{}, err 275 | } 276 | 277 | // for EKS/GKE, need to get the NodePort from clusterSvc 278 | // then it's able to proceed to add listener and handle firewall rules, etc. 279 | if err = r.provider.AssociateLB(request.NamespacedName, lbNamespacedName, clusterSvc); err != nil { 280 | // backoff a bit 281 | return reconcile.Result{RequeueAfter: time.Second * 1}, err 282 | } 283 | 284 | // it's reusing a LB, so availableLB is expected to carry loadbalancer info 285 | crObj.Status.Ref = lbNamespacedName.String() 286 | crObj.Status.LoadBalancer = availableLB.Status.LoadBalancer 287 | if portUpdated { 288 | // seems don't need a DeepCopy 289 | crObj.Spec.Ports = clusterSvc.Spec.Ports 290 | } 291 | err = r.Update(context.TODO(), crObj) 292 | if err != nil { 293 | return reconcile.Result{}, err 294 | } 295 | } else if err != nil { 296 | return reconcile.Result{}, err 297 | } 298 | 299 | // We don't care about update of cluster Service, so do nothing here 300 | return reconcile.Result{}, nil 301 | } 302 | 303 | func (pq *pendingQ) add(crName, lbName types.NamespacedName) { 304 | if pq.pendingLB == nil { 305 | pq.pendingLB = &lbName 306 | } 307 | pq.pendingCRs[crName] = struct{}{} 308 | } 309 | 310 | func (pq *pendingQ) remove(lbName types.NamespacedName) { 311 | pq.pendingLB = nil 312 | pq.pendingCRs = make(map[types.NamespacedName]struct{}) 313 | } 314 | 315 | func (pq *pendingQ) hasCR(crName types.NamespacedName) bool { 316 | _, ok := pq.pendingCRs[crName] 317 | return ok 318 | } 319 | 320 | func (pq *pendingQ) hasLB(lbName types.NamespacedName) bool { 321 | return pq.pendingLB != nil && *pq.pendingLB == lbName 322 | } 323 | 324 | func (pq *pendingQ) isEmpty() bool { 325 | return pq.pendingLB == nil 326 | } 327 | 328 | func containsString(slice []string, s string) bool { 329 | for _, item := range slice { 330 | if item == s { 331 | return true 332 | } 333 | } 334 | return false 335 | } 336 | 337 | func removeString(slice []string, s string) (result []string) { 338 | for _, item := range slice { 339 | if item == s { 340 | continue 341 | } 342 | result = append(result, item) 343 | } 344 | return 345 | } 346 | -------------------------------------------------------------------------------- /pkg/providers/eks.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "strings" 23 | 24 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 25 | "github.com/aws/aws-sdk-go/aws" 26 | "github.com/aws/aws-sdk-go/aws/awserr" 27 | "github.com/aws/aws-sdk-go/aws/endpoints" 28 | "github.com/aws/aws-sdk-go/aws/session" 29 | "github.com/aws/aws-sdk-go/service/ec2" 30 | "github.com/aws/aws-sdk-go/service/elb" 31 | corev1 "k8s.io/api/core/v1" 32 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 | "k8s.io/apimachinery/pkg/types" 34 | ) 35 | 36 | // Notes: 37 | // - don't use elbv2 package as the EKS LB service generated by default is not elbv2 instance 38 | 39 | // Refs: 40 | // https://docs.aws.amazon.com/sdk-for-go/api/service/elb/#New 41 | // https://docs.aws.amazon.com/sdk-for-go/api/service/elb/#example_ELB_DescribeLoadBalancers_shared00 42 | // https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#EC2.AuthorizeSecurityGroupIngress 43 | 44 | // for a EKS loadbalancer service, the corresponding ELS name is first section of hostname 45 | // e.g. a150664e6b12311e883b3061edd716de-2116084625.us-west-2.elb.amazonaws.com 46 | // the ELS name is a150664e6b12311e883b3061edd716de 47 | 48 | // EKS stands for Elastic(Amazon) Kubernetes Service 49 | type EKS struct { 50 | elbClient *elb.ELB 51 | ec2Client *ec2.EC2 52 | 53 | // key is namespacedName of a LB Serivce, val is the service 54 | cacheMap map[types.NamespacedName]*corev1.Service 55 | cacheELB map[types.NamespacedName]*elb.LoadBalancerDescription 56 | 57 | // cr to LB is 1:1 mapping 58 | crToLB map[types.NamespacedName]types.NamespacedName 59 | // lb to CRD is 1:N mapping 60 | lbToCRs map[types.NamespacedName]nameSet 61 | // lbToPorts is keyed with ns/name of a LB, and valued with ports info it holds 62 | lbToPorts map[types.NamespacedName]int32Set 63 | 64 | capacityPerLB int 65 | } 66 | 67 | var _ LBProvider = &EKS{} 68 | 69 | func newEKSProvider() *EKS { 70 | // TODO(Huang-Wei): make aws credentials and regionID configurable 71 | sess := session.Must(session.NewSession(&aws.Config{ 72 | Region: aws.String(endpoints.UsWest2RegionID), 73 | })) 74 | return &EKS{ 75 | elbClient: elb.New(sess), 76 | ec2Client: ec2.New(sess), 77 | cacheMap: make(map[types.NamespacedName]*corev1.Service), 78 | cacheELB: make(map[types.NamespacedName]*elb.LoadBalancerDescription), 79 | crToLB: make(map[types.NamespacedName]types.NamespacedName), 80 | lbToCRs: make(map[types.NamespacedName]nameSet), 81 | lbToPorts: make(map[types.NamespacedName]int32Set), 82 | capacityPerLB: capacity, 83 | } 84 | } 85 | 86 | func (e *EKS) GetCapacityPerLB() int { 87 | return e.capacityPerLB 88 | } 89 | 90 | func (e *EKS) UpdateCache(key types.NamespacedName, lbSvc *corev1.Service) { 91 | if lbSvc == nil { 92 | delete(e.cacheMap, key) 93 | delete(e.cacheELB, key) 94 | return 95 | } 96 | if len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 97 | return 98 | } 99 | 100 | // handle ELB stuff 101 | hostname := lbSvc.Status.LoadBalancer.Ingress[0].Hostname 102 | elbName := strings.Split(strings.Split(hostname, ".")[0], "-")[0] 103 | if result, err := e.queryELB(elbName); err != nil { 104 | log.WithName("eks").Error(err, "cannot query ELB", "key", key, "elbName", elbName) 105 | } else { 106 | log.WithName("eks").Info("ELB obj is updated in local cache", "key", key, "elbName", elbName) 107 | e.cacheELB[key] = result 108 | } 109 | e.cacheMap[key] = lbSvc 110 | } 111 | 112 | func (e *EKS) NewService(sharedLB *kubeconv1alpha1.SharedLB) *corev1.Service { 113 | return &corev1.Service{ 114 | ObjectMeta: metav1.ObjectMeta{ 115 | Name: sharedLB.Name + SvcPostfix, 116 | Namespace: sharedLB.Namespace, 117 | }, 118 | Spec: corev1.ServiceSpec{ 119 | // NodePort is the solution we can come up with so far 120 | Type: corev1.ServiceTypeNodePort, 121 | Ports: sharedLB.Spec.Ports, 122 | Selector: sharedLB.Spec.Selector, 123 | }, 124 | } 125 | } 126 | 127 | func (e *EKS) NewLBService() *corev1.Service { 128 | return &corev1.Service{ 129 | ObjectMeta: metav1.ObjectMeta{ 130 | Name: "lb-" + RandStringRunes(8), 131 | Namespace: namespace, 132 | Labels: map[string]string{"lb-template": ""}, 133 | }, 134 | Spec: corev1.ServiceSpec{ 135 | Type: corev1.ServiceTypeLoadBalancer, 136 | Selector: map[string]string{"app": "lb-placeholder"}, 137 | Ports: []corev1.ServicePort{ 138 | { 139 | Name: "tcp", 140 | Protocol: corev1.ProtocolTCP, 141 | Port: 33333, 142 | }, 143 | // EKS doesn't support UDP, so don't bother 144 | // { 145 | // Name: "UDP", 146 | // Protocol: corev1.ProtocolUDP, 147 | // Port: 33333, 148 | // }, 149 | }, 150 | }, 151 | } 152 | } 153 | 154 | func (e *EKS) GetAvailabelLB(clusterSvc *corev1.Service) *corev1.Service { 155 | // we leverage the randomness of golang "for range" when iterating 156 | OUTERLOOP: 157 | for lbKey, lbSvc := range e.cacheMap { 158 | if len(e.lbToCRs[lbKey]) >= e.capacityPerLB || len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 159 | continue 160 | } 161 | // must satisfy that all svc ports are not occupied in lbSvc 162 | for _, svcPort := range clusterSvc.Spec.Ports { 163 | if e.lbToPorts[lbKey] == nil { 164 | e.lbToPorts[lbKey] = int32Set{} 165 | } 166 | if _, ok := e.lbToPorts[lbKey][svcPort.Port]; ok { 167 | log.WithName("eks").Info(fmt.Sprintf("incoming service has port conflict with lbSvc %q on port %d", lbKey, svcPort.Port)) 168 | continue OUTERLOOP 169 | } 170 | } 171 | return lbSvc 172 | } 173 | return nil 174 | } 175 | 176 | func (e *EKS) AssociateLB(crName, lbName types.NamespacedName, clusterSvc *corev1.Service) error { 177 | // a) create LoadBalancer listener (create-load-balancer-listeners) 178 | // b) create inbound rules to security group (authorize-security-group-ingress) 179 | if clusterSvc != nil { 180 | if elbDesc := e.cacheELB[lbName]; elbDesc != nil { 181 | executed, err := e.createListeners(clusterSvc, elbDesc) 182 | if err != nil { 183 | return err 184 | } 185 | if executed { 186 | if err := e.createInboundRules(clusterSvc, elbDesc); err != nil { 187 | return err 188 | } 189 | } 190 | } 191 | // upon program starts, e.lbToPorts[lbName] can be nil 192 | if e.lbToPorts[lbName] == nil { 193 | e.lbToPorts[lbName] = int32Set{} 194 | } 195 | // update crToPorts 196 | for _, svcPort := range clusterSvc.Spec.Ports { 197 | e.lbToPorts[lbName][svcPort.Port] = struct{}{} 198 | } 199 | } 200 | 201 | // c) update internal cache 202 | // following code might be called multiple times, but shouldn't impact 203 | // performance a lot as all of them are O(1) operation 204 | _, ok := e.lbToCRs[lbName] 205 | if !ok { 206 | e.lbToCRs[lbName] = make(nameSet) 207 | } 208 | e.lbToCRs[lbName][crName] = struct{}{} 209 | e.crToLB[crName] = lbName 210 | log.WithName("eks").Info("AssociateLB", "cr", crName, "lb", lbName) 211 | return nil 212 | } 213 | 214 | // DeassociateLB is called by EKS finalizer to clean listeners 215 | // and inbound rules of security group 216 | func (e *EKS) DeassociateLB(crName types.NamespacedName, clusterSvc *corev1.Service) error { 217 | lbName, ok := e.crToLB[crName] 218 | if !ok { 219 | return nil 220 | } 221 | 222 | // a) remove LoadBalancer listener (delete-load-balancer-listeners) 223 | // b) remove inbound rules from security group (revoke-security-group-ingress) 224 | if elbDesc := e.cacheELB[lbName]; elbDesc != nil { 225 | if err := e.removeListeners(clusterSvc, elbDesc); err != nil { 226 | return err 227 | } 228 | if err := e.removeInboundRules(clusterSvc, elbDesc); err != nil { 229 | return err 230 | } 231 | } 232 | 233 | // c) update internal cache 234 | delete(e.crToLB, crName) 235 | delete(e.lbToCRs[lbName], crName) 236 | for _, svcPort := range clusterSvc.Spec.Ports { 237 | delete(e.lbToPorts[lbName], svcPort.Port) 238 | } 239 | log.WithName("eks").Info("DeassociateLB", "cr", crName, "lb", lbName) 240 | return nil 241 | } 242 | 243 | func (e *EKS) UpdateService(svc, lb *corev1.Service) (bool, bool) { 244 | lbName := types.NamespacedName{Name: lb.Name, Namespace: lb.Namespace} 245 | occupiedPorts := e.lbToPorts[lbName] 246 | if len(occupiedPorts) == 0 { 247 | occupiedPorts = int32Set{} 248 | } 249 | portUpdated := updatePort(svc, lb, occupiedPorts) 250 | // don't need to update externalIP 251 | return portUpdated, false 252 | } 253 | 254 | func (e *EKS) queryELB(elbName string) (*elb.LoadBalancerDescription, error) { 255 | if elbName == "" { 256 | return nil, errors.New("elbName cannot be empty") 257 | } 258 | input := &elb.DescribeLoadBalancersInput{ 259 | LoadBalancerNames: []*string{ 260 | aws.String(elbName), 261 | }, 262 | } 263 | result, err := e.elbClient.DescribeLoadBalancers(input) 264 | if err != nil { 265 | return nil, err 266 | } 267 | if len(result.LoadBalancerDescriptions) != 1 { 268 | return nil, fmt.Errorf("got %d elb.LoadBalancerDescription, but expected 1", len(result.LoadBalancerDescriptions)) 269 | } 270 | return result.LoadBalancerDescriptions[0], nil 271 | } 272 | 273 | // 1st return value means if it's executed 274 | // 2nd return value returns error if it's executed 275 | func (e *EKS) createListeners(clusterSvc *corev1.Service, elbDesc *elb.LoadBalancerDescription) (bool, error) { 276 | if clusterSvc == nil || elbDesc == nil { 277 | return false, errors.New("clusterSvc or elbDesc is nil") 278 | } 279 | listeners := make([]*elb.Listener, 0) 280 | for _, p := range clusterSvc.Spec.Ports { 281 | lowercasedProtocol := strings.ToLower(string(p.Protocol)) 282 | listener := elb.Listener{ 283 | InstancePort: aws.Int64(int64(p.NodePort)), 284 | InstanceProtocol: aws.String(lowercasedProtocol), 285 | LoadBalancerPort: aws.Int64(int64(p.Port)), 286 | Protocol: aws.String(lowercasedProtocol), 287 | // TODO(Huang-Wei): InstanceProtocol vs. Protocol? 288 | } 289 | // check if it exists in elbDesc 290 | if isListenerExisted(listener, elbDesc.ListenerDescriptions) { 291 | continue 292 | } 293 | listeners = append(listeners, &listener) 294 | } 295 | if len(listeners) == 0 { 296 | return false, nil 297 | } 298 | 299 | input := &elb.CreateLoadBalancerListenersInput{ 300 | Listeners: listeners, 301 | LoadBalancerName: elbDesc.LoadBalancerName, 302 | } 303 | _, err := e.elbClient.CreateLoadBalancerListeners(input) 304 | 305 | // tolerate if the listener exists in server side 306 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == elb.ErrCodeDuplicateListenerException { 307 | log.WithName("eks").Info("awserr", "code", aerr.Code()) 308 | return true, nil 309 | } 310 | 311 | return true, err 312 | } 313 | 314 | func isListenerExisted(l elb.Listener, listenerDescs []*elb.ListenerDescription) bool { 315 | for _, desc := range listenerDescs { 316 | e := desc.Listener 317 | if *l.InstancePort == *e.InstancePort && *l.InstanceProtocol == *e.InstanceProtocol && 318 | *l.LoadBalancerPort == *e.LoadBalancerPort && *l.Protocol == *e.Protocol { 319 | return true 320 | } 321 | } 322 | return false 323 | } 324 | 325 | func (e *EKS) createInboundRules(clusterSvc *corev1.Service, elbDesc *elb.LoadBalancerDescription) error { 326 | if clusterSvc == nil || elbDesc == nil { 327 | return errors.New("clusterSvc or elbDesc is nil") 328 | } 329 | 330 | sgStrs := elbDesc.SecurityGroups 331 | if len(sgStrs) == 0 { 332 | return errors.New("no security group is attached to the ELB") 333 | } 334 | 335 | ipPermissions := make([]*ec2.IpPermission, 0) 336 | for _, p := range clusterSvc.Spec.Ports { 337 | lowercasedProtocol := strings.ToLower(string(p.Protocol)) 338 | permission := ec2.IpPermission{ 339 | FromPort: aws.Int64(int64(p.Port)), 340 | IpProtocol: aws.String(lowercasedProtocol), 341 | IpRanges: []*ec2.IpRange{ 342 | { 343 | CidrIp: aws.String("0.0.0.0/0"), 344 | Description: aws.String("Generated by shared-loadblancer"), 345 | }, 346 | }, 347 | ToPort: aws.Int64(int64(p.Port)), 348 | } 349 | ipPermissions = append(ipPermissions, &permission) 350 | } 351 | if len(ipPermissions) == 0 { 352 | return nil 353 | } 354 | 355 | input := &ec2.AuthorizeSecurityGroupIngressInput{ 356 | // pick up the first security group 357 | // TODO(Huang-Wei): what if multiple security groups are found 358 | GroupId: sgStrs[0], 359 | IpPermissions: ipPermissions, 360 | } 361 | _, err := e.ec2Client.AuthorizeSecurityGroupIngress(input) 362 | // tolerate if the rules exist in server side 363 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "InvalidPermission.Duplicate" { 364 | return nil 365 | } 366 | return err 367 | } 368 | 369 | func (e *EKS) removeListeners(clusterSvc *corev1.Service, elbDesc *elb.LoadBalancerDescription) error { 370 | if clusterSvc == nil || elbDesc == nil { 371 | return errors.New("clusterSvc or elbDesc is nil") 372 | } 373 | ports := make([]*int64, len(clusterSvc.Spec.Ports)) 374 | for i, p := range clusterSvc.Spec.Ports { 375 | ports[i] = aws.Int64(int64(p.Port)) 376 | } 377 | input := &elb.DeleteLoadBalancerListenersInput{ 378 | LoadBalancerName: elbDesc.LoadBalancerName, 379 | LoadBalancerPorts: ports, 380 | } 381 | _, err := e.elbClient.DeleteLoadBalancerListeners(input) 382 | return err 383 | } 384 | 385 | func (e *EKS) removeInboundRules(clusterSvc *corev1.Service, elbDesc *elb.LoadBalancerDescription) error { 386 | if clusterSvc == nil || elbDesc == nil { 387 | return errors.New("clusterSvc or elbDesc is nil") 388 | } 389 | 390 | sgStrs := elbDesc.SecurityGroups 391 | if len(sgStrs) == 0 { 392 | return errors.New("no security group is attached to the ELB") 393 | } 394 | 395 | ipPermissions := make([]*ec2.IpPermission, 0) 396 | for _, p := range clusterSvc.Spec.Ports { 397 | lowercasedProtocol := strings.ToLower(string(p.Protocol)) 398 | permission := ec2.IpPermission{ 399 | FromPort: aws.Int64(int64(p.Port)), 400 | IpProtocol: aws.String(lowercasedProtocol), 401 | IpRanges: []*ec2.IpRange{ 402 | { 403 | CidrIp: aws.String("0.0.0.0/0"), 404 | Description: aws.String("Generated by shared-loadblancer"), 405 | }, 406 | }, 407 | ToPort: aws.Int64(int64(p.Port)), 408 | } 409 | ipPermissions = append(ipPermissions, &permission) 410 | } 411 | if len(ipPermissions) == 0 { 412 | return nil 413 | } 414 | 415 | input := &ec2.RevokeSecurityGroupIngressInput{ 416 | // pick up the first security group 417 | // TODO(Huang-Wei): what if multiple security groups are found 418 | GroupId: sgStrs[0], 419 | IpPermissions: ipPermissions, 420 | } 421 | _, err := e.ec2Client.RevokeSecurityGroupIngress(input) 422 | return err 423 | } 424 | -------------------------------------------------------------------------------- /pkg/providers/aks.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Shared LoadBalancer Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the Licensa. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apacha.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the Licensa. 15 | */ 16 | 17 | package providers 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | "reflect" 24 | "strconv" 25 | "strings" 26 | 27 | "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network" 28 | "github.com/Azure/go-autorest/autorest/azure/auth" 29 | "github.com/Azure/go-autorest/autorest/to" 30 | kubeconv1alpha1 "github.com/Huang-Wei/shared-loadbalancer/pkg/apis/kubecon/v1alpha1" 31 | corev1 "k8s.io/api/core/v1" 32 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 | "k8s.io/apimachinery/pkg/types" 34 | cloudprovider "k8s.io/cloud-provider" 35 | ) 36 | 37 | /* Pre-config steps: 38 | 39 | 1. az ad sp create-for-rbac --name 'sharedlb' 40 | 41 | 2. az role assignment create --assignee {appId from step 1} --role 4d97b98b-1d4f-4787-a291-c67834d212e7 42 | 43 | > `appid` can be get from step 1; 4d97b98b-1d4f-4787-a291-c67834d212e7 is a pre-created role in Azure for advanced network permissions. 44 | 45 | 3. setup following env variables: 46 | 47 | ``` 48 | export AZURE_TENANT_ID={tenant string from step 1} 49 | export AZURE_CLIENT_ID={appId string from step 1} 50 | export AZURE_CLIENT_SECRET={password string from step 1} 51 | export AZURE_SUBSCRIPTION_ID={can be get from Azure portal} 52 | ``` 53 | */ 54 | 55 | // Refs: 56 | // * https://github.com/Azure/azure-sdk-for-go 57 | // * https://github.com/Azure-Samples/azure-sdk-for-go-samples 58 | // * https://docs.microsoft.com/en-us/azure/azure-stack/user/azure-stack-version-profiles-go 59 | // * https://docs.microsoft.com/en-us/azure/aks/kubernetes-service-principal 60 | 61 | const ( 62 | loadBalancerMinimumPriority = 500 63 | loadBalancerMaximumPriority = 4096 64 | 65 | frontendIPConfigIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s" 66 | backendPoolIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/backendAddressPools/%s" 67 | ) 68 | 69 | var ( 70 | azureDefaultLBName = "kubernetes" 71 | ) 72 | 73 | // AKS stands for Azure(Microsoft) Kubernetes Service 74 | type AKS struct { 75 | subscriptionID string 76 | resGrpName string 77 | sgName string 78 | lbClient network.LoadBalancersClient 79 | sgClient network.SecurityGroupsClient 80 | pipClient network.PublicIPAddressesClient 81 | 82 | // key is namespacedName of a LB Serivce, val is the service 83 | cacheMap map[types.NamespacedName]*corev1.Service 84 | cachePIPMap map[types.NamespacedName]*network.PublicIPAddress 85 | cacheAzureDefaultLB *network.LoadBalancer 86 | 87 | // cr to LB is 1:1 mapping 88 | crToLB map[types.NamespacedName]types.NamespacedName 89 | // lb to CRD is 1:N mapping 90 | lbToCRs map[types.NamespacedName]nameSet 91 | // lbToPorts is keyed with ns/name of a LB, and valued with ports info it holds 92 | lbToPorts map[types.NamespacedName]int32Set 93 | 94 | capacityPerLB int 95 | } 96 | 97 | var _ LBProvider = &AKS{} 98 | 99 | func newAKSProvider() *AKS { 100 | // TODO(Huang-Wei): auto configure when running inside AKS cluster 101 | // for local testing, make sure following env variables are properly set: 102 | // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID 103 | subscriptionID := GetEnvVal("AZURE_SUBSCRIPTION_ID", "58de4ac8-a2d6-499b-b983-6f1c870d398e") 104 | authorizer, err := auth.NewAuthorizerFromEnvironment() 105 | if err != nil { 106 | panic(err) 107 | } 108 | 109 | aks := AKS{ 110 | subscriptionID: subscriptionID, 111 | // TODO(Huang-Wei): get it from node label kubernetes.azure.com/cluster 112 | resGrpName: GetEnvVal("RES_GRP_NAME", "MC_res-grp-1_wei-aks_eastus"), 113 | // TODO(Huang-Wei): read it from? 114 | sgName: GetEnvVal("SG_NAME", "aks-agentpool-37988249-nsg"), 115 | lbClient: network.NewLoadBalancersClient(subscriptionID), 116 | pipClient: network.NewPublicIPAddressesClient(subscriptionID), 117 | sgClient: network.NewSecurityGroupsClient(subscriptionID), 118 | cacheMap: make(map[types.NamespacedName]*corev1.Service), 119 | cachePIPMap: make(map[types.NamespacedName]*network.PublicIPAddress), 120 | crToLB: make(map[types.NamespacedName]types.NamespacedName), 121 | lbToCRs: make(map[types.NamespacedName]nameSet), 122 | lbToPorts: make(map[types.NamespacedName]int32Set), 123 | capacityPerLB: capacity, 124 | } 125 | aks.lbClient.Authorizer = authorizer 126 | aks.sgClient.Authorizer = authorizer 127 | aks.pipClient.Authorizer = authorizer 128 | 129 | return &aks 130 | } 131 | 132 | func (a *AKS) GetCapacityPerLB() int { 133 | return a.capacityPerLB 134 | } 135 | 136 | func (a *AKS) UpdateCache(key types.NamespacedName, lbSvc *corev1.Service) { 137 | if lbSvc == nil { 138 | delete(a.cacheMap, key) 139 | delete(a.cachePIPMap, key) 140 | return 141 | } 142 | if len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 143 | return 144 | } 145 | 146 | // handle Azure public/frontend ip 147 | pip := lbSvc.Status.LoadBalancer.Ingress[0].IP 148 | if result, err := a.queryPublicIP(pip, lbSvc); err != nil { 149 | log.WithName("aks").Error(err, "cannot query public ip", "pip", pip) 150 | } else { 151 | log.WithName("aks").Info("pip object is updated in local cache", "key", key, "pip", pip) 152 | a.cachePIPMap[key] = result 153 | } 154 | a.cacheMap[key] = lbSvc 155 | } 156 | 157 | func (a *AKS) NewService(sharedLB *kubeconv1alpha1.SharedLB) *corev1.Service { 158 | return &corev1.Service{ 159 | ObjectMeta: metav1.ObjectMeta{ 160 | Name: sharedLB.Name + SvcPostfix, 161 | Namespace: sharedLB.Namespace, 162 | }, 163 | Spec: corev1.ServiceSpec{ 164 | Type: corev1.ServiceTypeNodePort, 165 | Ports: sharedLB.Spec.Ports, 166 | Selector: sharedLB.Spec.Selector, 167 | }, 168 | } 169 | } 170 | 171 | func (a *AKS) NewLBService() *corev1.Service { 172 | return &corev1.Service{ 173 | ObjectMeta: metav1.ObjectMeta{ 174 | Name: "lb-" + RandStringRunes(8), 175 | Namespace: namespace, 176 | Labels: map[string]string{"lb-template": ""}, 177 | }, 178 | Spec: corev1.ServiceSpec{ 179 | Type: corev1.ServiceTypeLoadBalancer, 180 | Selector: map[string]string{"app": "lb-placeholder"}, 181 | Ports: []corev1.ServicePort{ 182 | { 183 | Name: "tcp", 184 | Protocol: corev1.ProtocolTCP, 185 | Port: 33333, 186 | }, 187 | }, 188 | }, 189 | } 190 | } 191 | 192 | func (a *AKS) GetAvailabelLB(clusterSvc *corev1.Service) *corev1.Service { 193 | // we leverage the randomness of golang "for range" when iterating 194 | OUTERLOOP: 195 | for lbKey, lbSvc := range a.cacheMap { 196 | if len(a.lbToCRs[lbKey]) >= a.capacityPerLB || len(lbSvc.Status.LoadBalancer.Ingress) == 0 { 197 | continue 198 | } 199 | // must satisfy that all svc ports are not occupied in lbSvc 200 | for _, svcPort := range clusterSvc.Spec.Ports { 201 | if a.lbToPorts[lbKey] == nil { 202 | a.lbToPorts[lbKey] = int32Set{} 203 | } 204 | if _, ok := a.lbToPorts[lbKey][svcPort.Port]; ok { 205 | log.WithName("aks").Info(fmt.Sprintf("incoming service has port conflict with lbSvc %q on port %d", lbKey, svcPort.Port)) 206 | continue OUTERLOOP 207 | } 208 | } 209 | return lbSvc 210 | } 211 | return nil 212 | } 213 | 214 | func (a *AKS) AssociateLB(crName, lbName types.NamespacedName, clusterSvc *corev1.Service) error { 215 | // a) create Azure LoadBalancer FrontendIP rule (az network lb rule create) 216 | // b) create Azure Network Security Group rule (az network nsg rule create) 217 | if clusterSvc != nil { 218 | pip, lbSvc := a.cachePIPMap[lbName], a.cacheMap[lbName] 219 | if pip != nil && lbSvc != nil { 220 | executed, err := a.reconcileLBRules(clusterSvc, lbSvc, true /* create */) 221 | if err != nil { 222 | return err 223 | } 224 | if executed { 225 | if err := a.reconcileSGRules(clusterSvc, lbSvc, true /* create */); err != nil { 226 | return err 227 | } 228 | } 229 | } 230 | // upon program starts, a.lbToPorts[lbName] can be nil 231 | if a.lbToPorts[lbName] == nil { 232 | a.lbToPorts[lbName] = int32Set{} 233 | } 234 | // update crToPorts 235 | for _, svcPort := range clusterSvc.Spec.Ports { 236 | a.lbToPorts[lbName][svcPort.Port] = struct{}{} 237 | } 238 | } 239 | 240 | // c) update internal cache 241 | // following code might be called multiple times, but shouldn't impact 242 | // performance a lot as all of them are O(1) operation 243 | _, ok := a.lbToCRs[lbName] 244 | if !ok { 245 | a.lbToCRs[lbName] = make(nameSet) 246 | } 247 | a.lbToCRs[lbName][crName] = struct{}{} 248 | a.crToLB[crName] = lbName 249 | log.WithName("aks").Info("AssociateLB", "cr", crName, "lb", lbName) 250 | return nil 251 | } 252 | 253 | // DeassociateLB is called by AKS finalizer to clean frontend ip rules 254 | // and network security group rules 255 | func (a *AKS) DeassociateLB(crName types.NamespacedName, clusterSvc *corev1.Service) error { 256 | lbName, ok := a.crToLB[crName] 257 | if !ok { 258 | return nil 259 | } 260 | 261 | // a) delete Azure LoadBalancer FrontendIP rule (az network lb rule delete) 262 | // b) delete Azure Network Security Group rule (az network nsg rule delete) 263 | if pip := a.cachePIPMap[lbName]; pip != nil { 264 | pip, lbSvc := a.cachePIPMap[lbName], a.cacheMap[lbName] 265 | if pip != nil && lbSvc != nil { 266 | executed, err := a.reconcileLBRules(clusterSvc, lbSvc, false /* delete */) 267 | if err != nil { 268 | return err 269 | } 270 | if executed { 271 | if err := a.reconcileSGRules(clusterSvc, lbSvc, false /* delete */); err != nil { 272 | return err 273 | } 274 | } 275 | } 276 | } 277 | 278 | // c) update internal cache 279 | delete(a.crToLB, crName) 280 | delete(a.lbToCRs[lbName], crName) 281 | for _, svcPort := range clusterSvc.Spec.Ports { 282 | delete(a.lbToPorts[lbName], svcPort.Port) 283 | } 284 | log.WithName("aks").Info("DeassociateLB", "cr", crName, "lb", lbName) 285 | return nil 286 | } 287 | 288 | func (a *AKS) UpdateService(svc, lb *corev1.Service) (bool, bool) { 289 | lbName := types.NamespacedName{Name: lb.Name, Namespace: lb.Namespace} 290 | occupiedPorts := a.lbToPorts[lbName] 291 | if len(occupiedPorts) == 0 { 292 | occupiedPorts = int32Set{} 293 | } 294 | portUpdated := updatePort(svc, lb, occupiedPorts) 295 | // don't need to update externalIP 296 | return portUpdated, false 297 | } 298 | 299 | func (a *AKS) getDefaultAzureLB() (*network.LoadBalancer, error) { 300 | azureLB, err := a.lbClient.Get(context.TODO(), a.resGrpName, azureDefaultLBName, "") 301 | if err != nil { 302 | return nil, err 303 | } 304 | return &azureLB, nil 305 | } 306 | 307 | func (a *AKS) queryPublicIP(pipName string, lbSvc *corev1.Service) (*network.PublicIPAddress, error) { 308 | if pipName == "" { 309 | return nil, errors.New("public ip cannot be empty") 310 | } 311 | 312 | lbFrontendIPConfigName := cloudprovider.DefaultLoadBalancerName(lbSvc) 313 | publicIP, err := a.pipClient.Get(context.TODO(), a.resGrpName, fmt.Sprintf("%s-%s", azureDefaultLBName, lbFrontendIPConfigName), "") 314 | return &publicIP, err 315 | } 316 | 317 | // 1st return value means if it's executed 318 | // 2nd return value returns error if it's executed 319 | func (a *AKS) reconcileLBRules(clusterSvc, lbSvc *corev1.Service, wantCreate bool) (bool, error) { 320 | azureLB, err := a.getDefaultAzureLB() 321 | if err != nil { 322 | // we don't create Azure LoadBalancer from scratch - it's owned by AKS cloud provider 323 | return false, err 324 | } 325 | lbFrontendIPConfigName := cloudprovider.DefaultLoadBalancerName(lbSvc) 326 | lbFrontendIPConfigID := a.getFrontendIPConfigID(*azureLB.Name, lbFrontendIPConfigName) 327 | // TODO(Huang-Wei): in AKS cloud provider code, it's using `clusterName` 328 | lbBackendPoolName := azureDefaultLBName 329 | lbBackendPoolID := a.getBackendPoolID(*azureLB.Name, lbBackendPoolName) 330 | 331 | // reconcile loadbalancing rules 332 | var updatedLBRules []network.LoadBalancingRule 333 | lbRules := make([]network.LoadBalancingRule, 0) 334 | for _, p := range clusterSvc.Spec.Ports { 335 | transportProto, _, _, err := getProtocolsFromKubernetesProtocol(p.Protocol) 336 | if err != nil { 337 | return false, err 338 | } 339 | lbRule := network.LoadBalancingRule{ 340 | // it's required to be consistent with AKS cloud provider naming pattern 341 | Name: to.StringPtr(fmt.Sprintf("%s-%s-%d", lbFrontendIPConfigName, p.Protocol, p.Port)), 342 | LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ 343 | Protocol: *transportProto, 344 | FrontendPort: to.Int32Ptr(p.Port), 345 | BackendPort: to.Int32Ptr(p.NodePort), 346 | IdleTimeoutInMinutes: to.Int32Ptr(4), 347 | EnableFloatingIP: to.BoolPtr(false), 348 | LoadDistribution: network.Default, 349 | FrontendIPConfiguration: &network.SubResource{ 350 | ID: &lbFrontendIPConfigID, 351 | }, 352 | BackendAddressPool: &network.SubResource{ 353 | ID: &lbBackendPoolID, 354 | }, 355 | // Probe is unnecessary 356 | }, 357 | } 358 | lbRules = append(lbRules, lbRule) 359 | } 360 | 361 | var needUpdate bool 362 | if wantCreate { 363 | // log.WithName("aks").Info("[DEBUG] before unionLBRules", "existing", len(*azureLB.LoadBalancingRules), "incoming", len(lbRules)) 364 | needUpdate, updatedLBRules = unionLBRules(*azureLB.LoadBalancingRules, lbRules) 365 | // log.WithName("aks").Info("[DEBUG] after unionLBRules", "existing", len(*azureLB.LoadBalancingRules), "updatedLBRules", len(updatedLBRules)) 366 | } else { 367 | // log.WithName("aks").Info("[DEBUG] before subtractLBRules", "existing", len(*azureLB.LoadBalancingRules), "incoming", len(lbRules)) 368 | needUpdate, updatedLBRules = subtractLBRules(*azureLB.LoadBalancingRules, lbRules) 369 | // log.WithName("aks").Info("[DEBUG] after subtractLBRules", "existing", len(*azureLB.LoadBalancingRules), "updatedLBRules", len(updatedLBRules)) 370 | } 371 | 372 | if !needUpdate { 373 | log.WithName("aks").Info("No need to reconcile LB rules") 374 | return false, nil 375 | } 376 | // create or update LB 377 | azureLB.LoadBalancingRules = &updatedLBRules 378 | _, err = a.lbClient.CreateOrUpdate(context.TODO(), a.resGrpName, *azureLB.Name, *azureLB) 379 | return true, err 380 | } 381 | 382 | func (a *AKS) reconcileSGRules(clusterSvc, lbSvc *corev1.Service, wantCreate bool) error { 383 | sg, err := a.sgClient.Get(context.TODO(), a.resGrpName, a.sgName, "") 384 | if err != nil { 385 | return err 386 | } 387 | lbFrontendIPConfigName := cloudprovider.DefaultLoadBalancerName(lbSvc) 388 | 389 | // reconcile securitygroup rules (only care about inbound rules) 390 | var updatedSGRules []network.SecurityRule 391 | sgRules := make([]network.SecurityRule, 0) 392 | for _, p := range clusterSvc.Spec.Ports { 393 | _, securityProto, _, err := getProtocolsFromKubernetesProtocol(p.Protocol) 394 | if err != nil { 395 | return err 396 | } 397 | if err != nil { 398 | return err 399 | } 400 | sgRule := network.SecurityRule{ 401 | Name: to.StringPtr(fmt.Sprintf("%s-%s-%d-Internet", lbFrontendIPConfigName, p.Protocol, p.Port)), 402 | SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ 403 | Protocol: *securityProto, 404 | // Priority: to.Int32Ptr(nextPriority), 405 | SourceAddressPrefix: to.StringPtr("Internet"), 406 | SourcePortRange: to.StringPtr("*"), 407 | DestinationAddressPrefix: to.StringPtr("*"), 408 | DestinationPortRange: to.StringPtr(strconv.Itoa(int(p.NodePort))), 409 | Access: network.SecurityRuleAccessAllow, 410 | Direction: network.SecurityRuleDirectionInbound, 411 | }, 412 | } 413 | sgRules = append(sgRules, sgRule) 414 | } 415 | var needUpdate bool 416 | if wantCreate { 417 | // log.WithName("aks").Info("[DEBUG] before unionSGRules", "existing", len(*sg.SecurityRules), "incoming", len(sgRules)) 418 | needUpdate, updatedSGRules = unionSGRules(*sg.SecurityRules, sgRules) 419 | // log.WithName("aks").Info("[DEBUG] after unionSGRules", "existing", len(*sg.SecurityRules), "updatedSGRules", len(updatedSGRules)) 420 | } else { 421 | // log.WithName("aks").Info("[DEBUG] before subtractSGRules", "existing", len(*sg.SecurityRules), "incoming", len(sgRules)) 422 | needUpdate, updatedSGRules = subtractSGRules(*sg.SecurityRules, sgRules) 423 | // log.WithName("aks").Info("[DEBUG] after subtractSGRules", "existing", len(*sg.SecurityRules), "updatedSGRules", len(updatedSGRules)) 424 | } 425 | 426 | if !needUpdate { 427 | log.WithName("aks").Info("No need to reconcile SG inbound rules") 428 | return nil 429 | } 430 | for i := range updatedSGRules { 431 | rule := updatedSGRules[i] 432 | if rule.Priority != nil { 433 | continue 434 | } 435 | nextPriority, err := getNextAvailablePriority(updatedSGRules) 436 | if err != nil { 437 | return err 438 | } 439 | rule.Priority = to.Int32Ptr(nextPriority) 440 | } 441 | // create or update SG 442 | sg.SecurityRules = &updatedSGRules 443 | _, err = a.sgClient.CreateOrUpdate(context.TODO(), a.resGrpName, a.sgName, sg) 444 | return err 445 | } 446 | 447 | func (a *AKS) getFrontendIPConfigID(lbName, backendPoolName string) string { 448 | return fmt.Sprintf( 449 | frontendIPConfigIDTemplate, 450 | a.subscriptionID, 451 | a.resGrpName, 452 | lbName, 453 | backendPoolName) 454 | } 455 | 456 | func (a *AKS) getBackendPoolID(lbName, backendPoolName string) string { 457 | return fmt.Sprintf( 458 | backendPoolIDTemplate, 459 | a.subscriptionID, 460 | a.resGrpName, 461 | lbName, 462 | backendPoolName) 463 | } 464 | 465 | func unionLBRules(existingRules, expectedRules []network.LoadBalancingRule) (bool, []network.LoadBalancingRule) { 466 | var needUpdate bool 467 | toAddRules := make([]network.LoadBalancingRule, 0) 468 | for _, expected := range expectedRules { 469 | if !findRule(existingRules, expected) { 470 | needUpdate = true 471 | toAddRules = append(toAddRules, expected) 472 | } 473 | } 474 | return needUpdate, append(existingRules, toAddRules...) 475 | } 476 | 477 | func subtractLBRules(existingRules, unexpectedRules []network.LoadBalancingRule) (bool, []network.LoadBalancingRule) { 478 | var needUpdate bool 479 | for i := len(existingRules) - 1; i >= 0; i-- { 480 | if findRule(unexpectedRules, existingRules[i]) { 481 | needUpdate = true 482 | existingRules = append(existingRules[:i], existingRules[i+1:]...) 483 | } 484 | } 485 | return needUpdate, existingRules 486 | } 487 | 488 | func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule) bool { 489 | for _, existingRule := range rules { 490 | if strings.EqualFold(to.String(existingRule.Name), to.String(rule.Name)) && 491 | equalLoadBalancingRulePropertiesFormat(existingRule.LoadBalancingRulePropertiesFormat, rule.LoadBalancingRulePropertiesFormat) { 492 | return true 493 | } 494 | } 495 | return false 496 | } 497 | 498 | // equalLoadBalancingRulePropertiesFormat checks whether the provided LoadBalancingRulePropertiesFormat are equal. 499 | func equalLoadBalancingRulePropertiesFormat(s, t *network.LoadBalancingRulePropertiesFormat) bool { 500 | if s == nil || t == nil { 501 | return false 502 | } 503 | 504 | return reflect.DeepEqual(s.Protocol, t.Protocol) && 505 | reflect.DeepEqual(s.FrontendIPConfiguration, t.FrontendIPConfiguration) && 506 | reflect.DeepEqual(s.BackendAddressPool, t.BackendAddressPool) && 507 | reflect.DeepEqual(s.LoadDistribution, t.LoadDistribution) && 508 | reflect.DeepEqual(s.FrontendPort, t.FrontendPort) && 509 | reflect.DeepEqual(s.BackendPort, t.BackendPort) && 510 | reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) && 511 | reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes) 512 | } 513 | 514 | func unionSGRules(existingRules, expectedRules []network.SecurityRule) (bool, []network.SecurityRule) { 515 | var needUpdate bool 516 | toAddRules := make([]network.SecurityRule, 0) 517 | for _, expected := range expectedRules { 518 | if !findSecurityRule(existingRules, expected) { 519 | needUpdate = true 520 | toAddRules = append(toAddRules, expected) 521 | } 522 | } 523 | return needUpdate, append(existingRules, toAddRules...) 524 | } 525 | 526 | func subtractSGRules(existingRules, unexpectedRules []network.SecurityRule) (bool, []network.SecurityRule) { 527 | var needUpdate bool 528 | for i := len(existingRules) - 1; i >= 0; i-- { 529 | if findSecurityRule(unexpectedRules, existingRules[i]) { 530 | needUpdate = true 531 | existingRules = append(existingRules[:i], existingRules[i+1:]...) 532 | } 533 | } 534 | return needUpdate, existingRules 535 | } 536 | 537 | // returns the equivalent LoadBalancerRule, SecurityRule and LoadBalancerProbe 538 | // protocol types for the given Kubernetes protocol type. 539 | func getProtocolsFromKubernetesProtocol(protocol corev1.Protocol) (*network.TransportProtocol, *network.SecurityRuleProtocol, *network.ProbeProtocol, error) { 540 | var transportProto network.TransportProtocol 541 | var securityProto network.SecurityRuleProtocol 542 | var probeProto network.ProbeProtocol 543 | 544 | switch protocol { 545 | case corev1.ProtocolTCP: 546 | transportProto = network.TransportProtocolTCP 547 | securityProto = network.SecurityRuleProtocolTCP 548 | probeProto = network.ProbeProtocolTCP 549 | return &transportProto, &securityProto, &probeProto, nil 550 | case corev1.ProtocolUDP: 551 | transportProto = network.TransportProtocolUDP 552 | securityProto = network.SecurityRuleProtocolUDP 553 | return &transportProto, &securityProto, nil, nil 554 | default: 555 | return &transportProto, &securityProto, &probeProto, fmt.Errorf("only TCP and UDP are supported for Azure LoadBalancers") 556 | } 557 | } 558 | 559 | // This compares rule's Name, Protocol, SourcePortRange, DestinationPortRange, SourceAddressPrefix, Access, and Direction. 560 | func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) bool { 561 | for _, existingRule := range rules { 562 | if !strings.EqualFold(to.String(existingRule.Name), to.String(rule.Name)) { 563 | continue 564 | } 565 | if existingRule.Protocol != rule.Protocol { 566 | continue 567 | } 568 | if !strings.EqualFold(to.String(existingRule.SourcePortRange), to.String(rule.SourcePortRange)) { 569 | continue 570 | } 571 | if !strings.EqualFold(to.String(existingRule.DestinationPortRange), to.String(rule.DestinationPortRange)) { 572 | continue 573 | } 574 | if !strings.EqualFold(to.String(existingRule.SourceAddressPrefix), to.String(rule.SourceAddressPrefix)) { 575 | continue 576 | } 577 | if existingRule.Access != rule.Access { 578 | continue 579 | } 580 | if existingRule.Direction != rule.Direction { 581 | continue 582 | } 583 | return true 584 | } 585 | return false 586 | } 587 | 588 | // This returns the next available rule priority level for a given set of security rules. 589 | func getNextAvailablePriority(rules []network.SecurityRule) (int32, error) { 590 | var smallest int32 = loadBalancerMinimumPriority 591 | var spread int32 = 1 592 | 593 | outer: 594 | for smallest < loadBalancerMaximumPriority { 595 | for _, rule := range rules { 596 | if rule.Priority == nil { 597 | continue 598 | } 599 | if *rule.Priority == smallest { 600 | smallest += spread 601 | continue outer 602 | } 603 | } 604 | // no one else had it 605 | return smallest, nil 606 | } 607 | 608 | return -1, fmt.Errorf("securityGroup priorities are exhausted") 609 | } 610 | --------------------------------------------------------------------------------