├── .dockerignore ├── doc ├── images │ ├── ecmp-scheme.png │ ├── bgp-for-ecmp.png │ ├── simple-scheme.png │ ├── bgp-for-ecmp-example.png │ └── bgp-for-ecmp-single.png ├── examples │ ├── bird-rr2.cfg │ ├── bird-core.cfg │ ├── bird-node1.cfg │ ├── bird-tor1.cfg │ ├── bird-tor2.cfg │ └── bird-rr1.cfg ├── auto-allocation.md ├── simple-deployment-scheme.md ├── fail-over-optimization.md ├── operating-modes.md └── ecmp-load-balancing.md ├── .gitignore ├── helm-chart ├── externalipcontroller-0.1.0.tgz └── externalipcontroller │ ├── templates │ ├── ipclaimpool.yaml │ ├── daemonset.yaml │ ├── deployment.yaml │ ├── NOTES.txt │ └── _helpers.tpl │ ├── values.yaml │ ├── .helmignore │ └── Chart.yaml ├── Dockerfile ├── scripts ├── config.sh ├── kube.sh ├── import.sh ├── push.sh └── dind.sh ├── examples ├── ip-pool.yml ├── claims │ ├── scheduler.yaml │ └── controller.yaml ├── nginx.yaml └── simple │ └── externalipcontroller.yaml ├── pkg ├── utils │ ├── tests_test.go │ └── tests.go ├── scheduler │ ├── node_filter.go │ ├── scheduler_test.go │ └── scheduler.go ├── workqueue │ ├── workqueue_test.go │ └── workqueue.go ├── extensions │ ├── register.go │ ├── types_test.go │ ├── testing │ │ └── client.go │ ├── types.go │ └── client.go ├── controller_test.go ├── claimcontroller │ ├── controller_test.go │ └── controller.go ├── netutils │ └── netutils.go └── controller.go ├── .travis.yml ├── cmd ├── app │ ├── root.go │ ├── naive.go │ ├── controller.go │ ├── options.go │ └── scheduler.go └── ipmanager │ └── main.go ├── test ├── e2e │ ├── basic_suite_test.go │ └── utils │ │ └── utils.go └── integration │ ├── network_suite_test.go │ └── network_test.go ├── glide.yaml ├── README.md ├── Makefile ├── LICENSE └── glide.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | test/* 2 | examples/* -------------------------------------------------------------------------------- /doc/images/ecmp-scheme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/k8s-externalipcontroller/master/doc/images/ecmp-scheme.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ipcontroller 2 | ipcontroller.tar 3 | test.test 4 | vendor/ 5 | _output/ 6 | workdir/ 7 | 8 | *.complete 9 | -------------------------------------------------------------------------------- /doc/images/bgp-for-ecmp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/k8s-externalipcontroller/master/doc/images/bgp-for-ecmp.png -------------------------------------------------------------------------------- /doc/images/simple-scheme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/k8s-externalipcontroller/master/doc/images/simple-scheme.png -------------------------------------------------------------------------------- /doc/images/bgp-for-ecmp-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/k8s-externalipcontroller/master/doc/images/bgp-for-ecmp-example.png -------------------------------------------------------------------------------- /doc/images/bgp-for-ecmp-single.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/k8s-externalipcontroller/master/doc/images/bgp-for-ecmp-single.png -------------------------------------------------------------------------------- /helm-chart/externalipcontroller-0.1.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/k8s-externalipcontroller/master/helm-chart/externalipcontroller-0.1.0.tgz -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/ipclaimpool.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ippool }} 2 | {{ .Values.ippool | include "ipclaimpool" }} 3 | {{ end }} 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.4 2 | MAINTAINER Dmitry Shulyak 3 | LABEL Name="k8s-externalipcontroller" Version="0.1" 4 | 5 | COPY _output/ipmanager /usr/sbin/ 6 | 7 | CMD ["sh", "-c", "/usr/sbin/ipmanager n --alsologtostderr=true --v=4 --iface=${HOST_INTERFACE}"] 8 | -------------------------------------------------------------------------------- /scripts/config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o xtrace 7 | 8 | NUM_NODES=${NUM_NODES:-2} 9 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 10 | WORKDIRECTORY=${WORKDIRECTORY:-"/tmp/kube"} 11 | mkdir -p "$WORKDIRECTORY" 12 | -------------------------------------------------------------------------------- /examples/ip-pool.yml: -------------------------------------------------------------------------------- 1 | apiVersion: ipcontroller.ext/v1 2 | kind: IpClaimPool 3 | metadata: 4 | name: test-pool 5 | spec: 6 | cidr: 192.168.0.248/29 7 | ranges: 8 | - - 192.168.0.249 9 | - 192.168.0.250 10 | - - 192.168.0.252 11 | - 192.168.0.253 12 | -------------------------------------------------------------------------------- /pkg/utils/tests_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestEventualCondition(t *testing.T) { 11 | fakeTest := &testing.T{} 12 | assert.False(t, 13 | EventualCondition(fakeTest, time.Millisecond*100, func() bool { 14 | return 2 == 3 15 | }, "test"), 16 | ) 17 | } 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: go 3 | services: 4 | - docker 5 | env: 6 | - WORKDIRECTORY=$HOME/kube 7 | cache: 8 | directories: 9 | - $HOME/kube 10 | before_install: 11 | - mkdir -p _output/ 12 | - sudo apt-get install -y git 13 | - sudo apt-get install libpcap-dev 14 | install: 15 | - make get-deps 16 | script: make test 17 | after_success: 18 | - ./scripts/push.sh 19 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for externalipcontroller. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | replicaCount: 2 5 | image: 6 | repository: mirantis/k8s-externalipcontroller 7 | tag: latest 8 | pullPolicy: IfNotPresent 9 | scheduler: 10 | name: claimscheduler 11 | network_mask: 24 12 | controller: 13 | name: claimcontroller 14 | interface: docker0 15 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: Configure External IPs on k8s worker node(s) to provide IP connectivity. 3 | name: externalipcontroller 4 | version: 0.1.0 5 | keywords: 6 | - externalip 7 | - external-ip 8 | - externalipcontroller 9 | - external-ip-controller 10 | sources: 11 | - https://github.com/Mirantis/k8s-externalipcontroller 12 | maintainers: 13 | - name: Dmitry Shulyak 14 | email: yashulyak@gmail.com 15 | - name: Aleksei Kasatkin 16 | email: alekseyk.ru@gmail.com 17 | -------------------------------------------------------------------------------- /examples/claims/scheduler.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: claimscheduler 5 | spec: 6 | replicas: 2 7 | template: 8 | metadata: 9 | labels: 10 | app: claimscheduler 11 | spec: 12 | containers: 13 | - name: externalipcontroller 14 | image: mirantis/k8s-externalipcontroller 15 | imagePullPolicy: IfNotPresent 16 | command: 17 | - ipmanager 18 | - scheduler 19 | - --mask=24 20 | - --logtostderr 21 | - --v=5 22 | - --leader-elect=true 23 | - --monitor=1s 24 | - --nodefilter=fair 25 | -------------------------------------------------------------------------------- /examples/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: nginx 5 | spec: 6 | replicas: 2 7 | template: 8 | metadata: 9 | labels: 10 | app: nginx 11 | spec: 12 | containers: 13 | - name: nginx 14 | image: nginx 15 | imagePullPolicy: IfNotPresent 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: nginxsvc 21 | labels: 22 | app: nginx 23 | spec: 24 | type: NodePort 25 | ports: 26 | - port: 80 27 | protocol: TCP 28 | name: http 29 | - port: 443 30 | protocol: TCP 31 | name: https 32 | selector: 33 | app: nginx 34 | externalIPs: 35 | - 10.0.0.7 36 | - 10.0.0.8 -------------------------------------------------------------------------------- /examples/claims/controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: claimcontroller 5 | spec: 6 | template: 7 | metadata: 8 | labels: 9 | app: externalipcontroller 10 | spec: 11 | hostNetwork: true 12 | containers: 13 | - name: externalipcontroller 14 | image: mirantis/k8s-externalipcontroller 15 | imagePullPolicy: IfNotPresent 16 | securityContext: 17 | privileged: true 18 | command: 19 | - ipmanager 20 | - claimcontroller 21 | # iface is environment specific 22 | - --iface=docker0 23 | - --logtostderr 24 | - --v=5 25 | - --hb=500ms 26 | -------------------------------------------------------------------------------- /doc/examples/bird-rr2.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | debug protocols all; 3 | 4 | protocol device { 5 | scan time 2; 6 | } 7 | 8 | protocol bgp 'rack-2' { 9 | local as 65002; 10 | neighbor 10.211.2.254 port 179 as 65002; 11 | description "TOR-10.211.2.254"; 12 | multihop; 13 | rr client; 14 | import all; 15 | export all; 16 | next hop keep; 17 | source address 10.211.2.1; 18 | add paths; # For ECMP in BGP session with TOR 19 | } 20 | 21 | protocol bgp 'node-10.211.2.2' { 22 | local as 65002; 23 | neighbor 10.211.2.2 port 179 as 65002; 24 | description "node-10.211.2.2"; 25 | multihop; 26 | rr client; 27 | import all; 28 | export all; 29 | next hop keep; 30 | source address 10.211.2.1; 31 | } 32 | -------------------------------------------------------------------------------- /doc/examples/bird-core.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.254.254; 3 | 4 | debug protocols all; 5 | 6 | protocol kernel { 7 | learn; 8 | persist; 9 | scan time 2; 10 | import all; 11 | graceful restart; 12 | export all; 13 | merge paths; # For ECMP in routing table 14 | } 15 | 16 | protocol direct { 17 | interface "*"; 18 | } 19 | 20 | protocol device { 21 | scan time 2; 22 | } 23 | 24 | protocol bgp 'rack-1' { 25 | local as 65000; 26 | neighbor 192.168.192.6 as 65001; 27 | description "rack-1"; 28 | import all; 29 | export all; 30 | } 31 | 32 | protocol bgp 'rack-2' { 33 | local as 65000; 34 | neighbor 192.168.192.10 as 65002; 35 | description "rack-2"; 36 | import all; 37 | export all; 38 | } 39 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: {{ .Values.controller.name }} 5 | spec: 6 | template: 7 | metadata: 8 | labels: 9 | app: {{ .Chart.Name }} 10 | spec: 11 | hostNetwork: true 12 | containers: 13 | - name: {{ .Chart.Name }} 14 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 15 | imagePullPolicy: {{ .Values.image.pullPolicy }} 16 | securityContext: 17 | privileged: true 18 | command: 19 | - ipmanager 20 | - claimcontroller 21 | - --iface={{ .Values.controller.interface }} 22 | - --logtostderr 23 | - --v=5 24 | - --hb=500ms 25 | -------------------------------------------------------------------------------- /scripts/kube.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o xtrace 7 | 8 | CURDIR=$(dirname "${BASH_SOURCE[0]}") 9 | source "${CURDIR}/config.sh" 10 | DESIRED_TAG="v1.4.4" 11 | 12 | function fetch-kube { 13 | cd "${WORKDIRECTORY}" 14 | if [ ! -d "kubernetes" ]; then 15 | git clone https://github.com/kubernetes/kubernetes.git 16 | fi 17 | cd kubernetes/ 18 | local current_tag=$(git describe --tags) 19 | if [ "$current_tag" == "$DESIRED_TAG" ]; then 20 | echo "Assuming that sources are already built for tag $DESIRED_TAG. Exiting..." 21 | exit 0 22 | fi 23 | git checkout tags/v1.4.4 24 | go get -u github.com/jteeuwen/go-bindata/go-bindata 25 | make WHAT='cmd/hyperkube' 26 | make WHAT='cmd/kubectl' 27 | } 28 | 29 | fetch-kube 30 | -------------------------------------------------------------------------------- /doc/examples/bird-node1.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.1.2 3 | debug protocols all; 4 | 5 | protocol device { 6 | scan time 2; 7 | } 8 | 9 | listen bgp address 10.211.1.2 port 179; 10 | 11 | filter exported_by_bgp { 12 | if ( ifname = "lo" ) then { 13 | if net != 127.0.0.0/8 then accept; 14 | } 15 | reject; 16 | } 17 | 18 | protocol kernel { 19 | learn; 20 | persist; 21 | scan time 2; 22 | import all; 23 | graceful restart; 24 | export all; 25 | } 26 | 27 | protocol direct { 28 | debug all; 29 | interface "-docker*", "*"; 30 | } 31 | 32 | protocol bgp 'RR-10.211.1.1' { 33 | local as 65001; 34 | neighbor 10.211.1.1 port 179 as 65001; 35 | description "RR-10.211.1.1"; 36 | import all; 37 | export filter exported_by_bgp; 38 | next hop self; 39 | source address 10.211.1.2; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /examples/simple/externalipcontroller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: externalipcontroller 5 | spec: 6 | # only single replica allowed until we will add protection against fighting for 7 | # same ip, this agent will probably become daemonset at that point 8 | replicas: 1 9 | template: 10 | metadata: 11 | labels: 12 | app: externalipcontroller 13 | spec: 14 | hostNetwork: true 15 | containers: 16 | - name: externalipcontroller 17 | image: mirantis/k8s-externalipcontroller 18 | imagePullPolicy: IfNotPresent 19 | securityContext: 20 | privileged: true 21 | env: 22 | # TODO this is specific for my vagrant environment, configmap should be used here 23 | - name: HOST_INTERFACE 24 | value: eth0 25 | -------------------------------------------------------------------------------- /doc/examples/bird-tor1.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.1.254; 3 | debug protocols all; 4 | 5 | protocol kernel { 6 | learn; 7 | persist; 8 | scan time 2; 9 | import all; 10 | graceful restart; 11 | export all; 12 | merge paths; # For ECMP in routing table 13 | } 14 | 15 | protocol direct { 16 | interface "*"; 17 | } 18 | 19 | protocol device { 20 | scan time 2; 21 | } 22 | 23 | protocol bgp 'master-192.168.192.5' { 24 | local as 65001; 25 | neighbor 192.168.192.5 as 65000; 26 | description "master"; 27 | import all; 28 | export all; 29 | next hop self; 30 | } 31 | 32 | protocol bgp 'node-01-001' { 33 | local as 65001; 34 | neighbor 10.211.1.1 as 65001; 35 | description "RR-node-01-001"; 36 | multihop; 37 | import all; 38 | export all; 39 | next hop self; 40 | add paths; # For ECMP in BGP session with RR 41 | } 42 | -------------------------------------------------------------------------------- /doc/examples/bird-tor2.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.2.254; 3 | debug protocols all; 4 | 5 | protocol kernel { 6 | learn; 7 | persist; 8 | scan time 2; 9 | import all; 10 | graceful restart; 11 | export all; 12 | merge paths; # For ECMP in routing table 13 | } 14 | 15 | protocol direct { 16 | interface "*"; 17 | } 18 | 19 | protocol device { 20 | scan time 2; 21 | } 22 | 23 | protocol bgp 'master-192.168.192.9' { 24 | local as 65002; 25 | neighbor 192.168.192.9 as 65000; 26 | description "master"; 27 | import all; 28 | export all; 29 | next hop self; 30 | } 31 | 32 | protocol bgp 'node-02-001' { 33 | local as 65002; 34 | neighbor 10.211.2.1 as 65002; 35 | description "RR-node-02-001"; 36 | multihop; 37 | import all; 38 | export all; 39 | next hop self; 40 | add paths; # For ECMP in BGP session with RR 41 | } 42 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.scheduler.name }} 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | app: {{ .Values.scheduler.name }} 13 | spec: 14 | containers: 15 | - name: {{ .Chart.Name }} 16 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 17 | imagePullPolicy: {{ .Values.image.pullPolicy }} 18 | command: 19 | - ipmanager 20 | - scheduler 21 | - --mask={{ .Values.scheduler.network_mask }} 22 | - --logtostderr 23 | - --v=5 24 | - --leader-elect=true 25 | - --monitor=4s 26 | - --nodefilter=fair 27 | -------------------------------------------------------------------------------- /cmd/app/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import "github.com/spf13/cobra" 18 | 19 | var Root = &cobra.Command{ 20 | Use: "ipmanager", 21 | Short: "Application to manage IPs assignment to k8s services", 22 | Run: func(cmd *cobra.Command, args []string) {}, 23 | } 24 | -------------------------------------------------------------------------------- /test/e2e/basic_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package e2e 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestBasicFeatures(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Basic") 27 | } 28 | -------------------------------------------------------------------------------- /test/integration/network_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestNetwork(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Network") 27 | } 28 | -------------------------------------------------------------------------------- /scripts/import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o xtrace 7 | 8 | CURDIR=$(dirname "${BASH_SOURCE[0]}") 9 | source "${CURDIR}/config.sh" 10 | 11 | IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-externalipcontroller} 12 | IMAGE_TAG=${IMAGE_TAG:-latest} 13 | 14 | function import-image { 15 | echo "Export docker image and import it on a dind node dind_node_1" 16 | CONTAINERID="$(docker create "${IMAGE_REPO}":"${IMAGE_TAG}" bash)" 17 | mkdir -p _output/ 18 | docker export "${CONTAINERID}" > _output/ipcontroller.tar 19 | # TODO implement it as a provider (e.g source functions) 20 | 21 | for i in $(seq 1 "${NUM_NODES}"); do 22 | docker cp _output/ipcontroller.tar dind_node_"$i":/tmp 23 | docker exec -ti dind_node_"$i" docker import /tmp/ipcontroller.tar "${IMAGE_REPO}":"${IMAGE_TAG}" 24 | done 25 | echo "Finished copying docker image to dind nodes" 26 | } 27 | 28 | import-image 29 | -------------------------------------------------------------------------------- /doc/examples/bird-rr1.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | debug protocols all; 3 | 4 | protocol device { 5 | scan time 2; 6 | } 7 | 8 | protocol bgp 'rack-1' { 9 | local as 65001; 10 | neighbor 10.211.1.254 port 179 as 65001; 11 | description "TOR-10.211.1.254"; 12 | multihop; 13 | rr client; 14 | import all; 15 | export all; 16 | next hop keep; 17 | source address 10.211.1.1; 18 | add paths; # For ECMP in BGP session with TOR 19 | } 20 | 21 | protocol bgp 'node-10.211.1.2' { 22 | local as 65001; 23 | neighbor 10.211.1.2 port 179 as 65001; 24 | description "node-10.211.1.2"; 25 | multihop; 26 | rr client; 27 | import all; 28 | export all; 29 | next hop keep; 30 | source address 10.211.1.1; 31 | } 32 | 33 | protocol bgp 'node-10.211.1.3' { 34 | local as 65001; 35 | neighbor 10.211.1.3 port 179 as 65001; 36 | description "node-10.211.1.3"; 37 | multihop; 38 | rr client; 39 | import all; 40 | export all; 41 | next hop keep; 42 | source address 10.211.1.1; 43 | } 44 | -------------------------------------------------------------------------------- /scripts/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-} 8 | 9 | function push-to-docker { 10 | if [ "$TRAVIS_PULL_REQUEST_BRANCH" != "" ]; then 11 | echo "Processing PR $TRAVIS_PULL_REQUEST_BRANCH" 12 | exit 0 13 | fi 14 | 15 | docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" 16 | 17 | set -o xtrace 18 | TRAVIS_BRANCH=${TRAVIS_BRANCH:-} 19 | local branch=$TRAVIS_BRANCH 20 | echo "Using git branch $branch" 21 | 22 | if [ "$branch" == "master" ]; then 23 | echo "Pushing with tag - latest" 24 | docker push mirantis/k8s-externalipcontroller:latest; 25 | fi 26 | 27 | if [ "${branch:0:8}" == "release-" ]; then 28 | echo "Pushing from release branch with tag - $branch" 29 | docker tag mirantis/k8s-externalipcontroller mirantis/k8s-externalipcontroller:"$branch"; 30 | docker push mirantis/k8s-externalipcontroller:"$branch"; 31 | fi 32 | } 33 | 34 | push-to-docker 35 | -------------------------------------------------------------------------------- /cmd/ipmanager/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "log" 19 | 20 | "github.com/Mirantis/k8s-externalipcontroller/cmd/app" 21 | 22 | "k8s.io/kubernetes/pkg/util/flag" 23 | ) 24 | 25 | func main() { 26 | flag.InitFlags() 27 | if err := app.AppOpts.CheckFlags(); err != nil { 28 | log.Fatal(err) 29 | } 30 | if err := app.Root.Execute(); err != nil { 31 | log.Fatal(err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | For general information about External IP Controller application, see: 2 | https://github.com/Mirantis/k8s-externalipcontroller 3 | 4 | For more details about using External IP Controller application, see: 5 | https://github.com/Mirantis/k8s-externalipcontroller/blob/master/doc 6 | 7 | This chart allows running External IP Controller application in claims mode. 8 | For more details about application operational modes, see: 9 | https://github.com/Mirantis/k8s-externalipcontroller/blob/master/doc/operating-modes.md 10 | 11 | In order to auto-create IpClaimPool one needs to add ippool.cidr and ippool.ranges into Values 12 | where ippool.ranges is optional. 13 | definition example: 14 | 15 | ippool: 16 | cidr: 192.168.0.248/29 17 | ranges: 18 | - - 192.168.0.249 19 | - 192.168.0.250 20 | - - 192.168.0.252 21 | - 192.168.0.253 22 | 23 | For more details about auto allocation of external IPs, see: 24 | https://github.com/Mirantis/k8s-externalipcontroller/blob/master/doc/auto-allocation.md 25 | -------------------------------------------------------------------------------- /scripts/dind.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o xtrace 7 | 8 | CURDIR=$(dirname "${BASH_SOURCE[0]}") 9 | source "${CURDIR}/config.sh" 10 | 11 | DIND_IMAGE="k8s.io/kubernetes-dind" 12 | DIND_TAR="$WORKDIRECTORY/dind.tar" 13 | DIND_COMPATIBLE_COMMIT=${DIND_COMPATIBLE_COMMIT:-897ad95a8e0e1fe674ff81533d4198a3cecee41e} 14 | 15 | function prepare-dind-cluster { 16 | cd "${WORKDIRECTORY}"/kubernetes 17 | if [ ! -d "dind" ]; then 18 | git clone https://github.com/sttts/kubernetes-dind-cluster.git dind 19 | pushd dind &> /dev/null 20 | git checkout "$DIND_COMPATIBLE_COMMIT" 21 | popd &> /dev/null 22 | fi 23 | if [ -f "$DIND_TAR" ]; then 24 | docker import "$DIND_TAR" "$DIND_IMAGE" 25 | fi 26 | NUM_NODES="${NUM_NODES}" dind/dind-down-cluster.sh 27 | NUM_NODES="${NUM_NODES}" dind/dind-up-cluster.sh 28 | for i in $(seq 1 "${NUM_NODES}"); do 29 | docker exec -ti dind_node_"$i" ip l set docker0 promisc on 30 | done 31 | echo "Saving $DIND_IMAGE into tar $DIND_TAR" 32 | docker save "$DIND_IMAGE" > "$DIND_TAR" 33 | } 34 | 35 | prepare-dind-cluster 36 | -------------------------------------------------------------------------------- /pkg/utils/tests.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/stretchr/testify/assert" 21 | ) 22 | 23 | func EventualCondition(t assert.TestingT, wait time.Duration, comp assert.Comparison, msgAndArgs ...interface{}) bool { 24 | after := time.After(wait) 25 | for { 26 | select { 27 | case <-after: 28 | return assert.Condition(t, comp, msgAndArgs...) 29 | case <-time.Tick(10 * time.Millisecond): 30 | if comp() { 31 | return true 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/Mirantis/k8s-externalipcontroller 2 | import: 3 | - package: github.com/golang/glog 4 | - package: github.com/vishvananda/netlink 5 | - package: k8s.io/client-go 6 | version: v1.5.0 7 | subpackages: 8 | - 1.5/kubernetes 9 | - 1.5/pkg/api 10 | - 1.5/pkg/runtime 11 | - 1.5/pkg/watch 12 | - 1.5/rest 13 | - 1.5/tools/cache 14 | - package: github.com/onsi/ginkgo 15 | - package: github.com/onsi/gomega 16 | version: v1.0 17 | - package: github.com/coreos/etcd 18 | version: v3.1.0-rc.0 19 | subpackages: 20 | - client 21 | - package: github.com/stretchr/testify 22 | version: v1.1.4 23 | - package: github.com/pmezard/go-difflib 24 | version: v1.0.0 25 | subpackages: 26 | - difflib 27 | - package: k8s.io/kubernetes 28 | version: v1.6.0-alpha.0 29 | subpackages: 30 | - pkg/client/leaderelection 31 | - pkg/apis/componentconfig 32 | - pkg/client/unversioned/remotecommand 33 | - pkg/util/flag 34 | - pkg/client/record 35 | - pkg/client/restclient 36 | - pkg/client/clientset_generated/internalclientset 37 | - pkg/client/leaderelection/resourcelock 38 | - package: github.com/spf13/cobra 39 | - package: github.com/google/gopacket 40 | version: v1.1.12 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | External ip controller [![Build Status](https://travis-ci.org/Mirantis/k8s-externalipcontroller.svg?branch=master)](https://travis-ci.org/Mirantis/k8s-externalipcontroller) 2 | ====================== 3 | 4 | ## Introduction 5 | 6 | One of the possible ways to expose k8s services on a bare metal deployments is 7 | using External IPs. Each node runs a kube-proxy process which programs `iptables` 8 | rules to trap access to External IPs and redirect them to the correct backends. 9 | 10 | So in order to access k8s service from the outside we just need to route public 11 | traffic to one of k8s worker nodes which has `kube-proxy` running and thus has 12 | needed `iptables` rules for External IPs. 13 | 14 | ## Proposed solution 15 | 16 | External IP controller is k8s application which is deployed on top of k8s 17 | cluster and which configures External IPs on k8s worker node(s) to provide 18 | IP connectivity. 19 | 20 | ## Demo 21 | 22 | [![asciicast](https://asciinema.org/a/95449.png)](https://asciinema.org/a/95449) 23 | 24 | How to run tests 25 | ================ 26 | 27 | Install dependencies and prepare kube dind cluster 28 | ``` 29 | make get-deps 30 | ``` 31 | 32 | Build necessary images and run tests 33 | ``` 34 | make test 35 | ``` 36 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- $name := default .Chart.Name .Values.nameOverride -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 16 | {{- end -}} 17 | 18 | {{/* 19 | Create IpClaimPool resource to provide IP pool for the IP claim allocator. 20 | */}} 21 | {{- define "ipclaimpool" -}} 22 | apiVersion: ipcontroller.ext/v1 23 | kind: IpClaimPool 24 | metadata: 25 | name: default-pool 26 | spec: 27 | cidr: {{ .cidr }} 28 | {{ if .ranges }} 29 | {{ . | include "ipranges" }} 30 | {{- end -}} 31 | {{- end -}} 32 | 33 | {{/* 34 | Define IP ranges list for IpClaimPool resource. 35 | */}} 36 | {{- define "ipranges" -}} 37 | ranges: 38 | {{- range .ranges }} 39 | - - {{ index . 0 | quote }} 40 | - {{ index . 1 | quote }} 41 | {{- end }} 42 | {{- end -}} 43 | -------------------------------------------------------------------------------- /pkg/scheduler/node_filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scheduler 16 | 17 | import ( 18 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 19 | 20 | "github.com/golang/glog" 21 | ) 22 | 23 | func (s *ipClaimScheduler) getFairNode(ipnodes []*extensions.IpNode) *extensions.IpNode { 24 | counter := make(map[string]int) 25 | for _, key := range s.claimStore.ListKeys() { 26 | obj, _, err := s.claimStore.GetByKey(key) 27 | claim := obj.(*extensions.IpClaim) 28 | if err != nil { 29 | glog.Errorln(err) 30 | continue 31 | } 32 | if claim.Spec.NodeName == "" { 33 | continue 34 | } 35 | counter[claim.Spec.NodeName]++ 36 | } 37 | var min *extensions.IpNode 38 | minCount := -1 39 | for _, node := range ipnodes { 40 | count := counter[node.Metadata.Name] 41 | if minCount == -1 || count < minCount { 42 | minCount = count 43 | min = node 44 | } 45 | } 46 | return min 47 | } 48 | 49 | func (s *ipClaimScheduler) getFirstAliveNode(ipnodes []*extensions.IpNode) *extensions.IpNode { 50 | return ipnodes[0] 51 | } 52 | -------------------------------------------------------------------------------- /doc/auto-allocation.md: -------------------------------------------------------------------------------- 1 | Auto allocation of external IPs 2 | =============================== 3 | 4 | Now users can utilize auto allocation feature. 5 | 6 | In order to do that they must first provide 7 | IP pool for the allocator by creation of IpClaimPool resources. 8 | Here is example description for one in yaml format 9 | 10 | ```yaml 11 | apiVersion: ipcontroller.ext/v1 12 | kind: IpClaimPool 13 | metadata: 14 | name: test-pool 15 | spec: 16 | cidr: 192.168.0.248/29 17 | ranges: 18 | - - 192.168.0.249 19 | - 192.168.0.250 20 | - - 192.168.0.252 21 | - 192.168.0.253 22 | ``` 23 | 24 | Few words about the Spec: CIDR is mandatory to supply. 25 | Ranges can be omitted (range of the whole network defined in "CIDR" field is used then). 26 | Exclusion of particular addresses is done via specifying multiple ranges. 27 | In example above address 192.168.0.251 is not processed by the allocator. 28 | 29 | In order to enable auto allocation for particular services users 30 | must annotate them with following key-value pair - "external-ip = auto" 31 | either on creation or while the service is running (via `kubectl annotate`). 32 | After that there will be allocated exactly one external IP address for the service 33 | and IP claim will be created. Allocated address is then stored in 'allocated' field of pool 34 | object. 35 | 36 | Users can create several IP pools. Allocator will process all of them in the same manner 37 | (looking for first available IP) and with no regard for order, that is users cannot control 38 | which pool an address is fetched from. 39 | 40 | When service is deleted allocation is freed and returned to the pool of usable addresses 41 | again. Corresponding IP claim is removed. 42 | 43 | Here is a brief demo of the functionality. 44 | 45 | [![asciicast](https://asciinema.org/a/6uyrkfn66nufzpuhrwt1veshb.png)](https://asciinema.org/a/6uyrkfn66nufzpuhrwt1veshb) 46 | 47 | -------------------------------------------------------------------------------- /pkg/workqueue/workqueue_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package workqueue 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestQueue(t *testing.T) { 23 | queue := NewQueue() 24 | queue.Add(1) 25 | queue.Add(2) 26 | queue.Remove(1) 27 | item, _ := queue.Get() 28 | if citem := item.(int); citem != 2 { 29 | t.Errorf("item expected to be 2 - %v", citem) 30 | } 31 | queue.Close() 32 | item, closed := queue.Get() 33 | if item != nil { 34 | t.Errorf("expected to return nil if empty") 35 | } 36 | if !closed { 37 | t.Errorf("queue expected to be closed") 38 | } 39 | } 40 | 41 | func TestProcessQueue(t *testing.T) { 42 | queue := ProcessingQueue{NewQueue()} 43 | queue.Add(1) 44 | queue.Add(2) 45 | queue.Process(func(item interface{}) error { 46 | if citem := item.(int); citem != 1 { 47 | t.Errorf("item expected to be 1 - %v - %v", citem, queue.queue) 48 | } 49 | return fmt.Errorf("test") 50 | }) 51 | queue.Process(func(item interface{}) error { 52 | if citem := item.(int); citem != 2 { 53 | t.Errorf("item expected to be 2 - %v - %v", citem, queue.queue) 54 | } 55 | return nil 56 | }) 57 | queue.Process(func(item interface{}) error { 58 | if citem := item.(int); citem != 1 { 59 | t.Errorf("item expected to be 1 - %v - %v", citem, queue.queue) 60 | } 61 | return nil 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /cmd/app/naive.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "os" 19 | 20 | "github.com/golang/glog" 21 | "github.com/spf13/cobra" 22 | "k8s.io/client-go/1.5/rest" 23 | "k8s.io/client-go/1.5/tools/clientcmd" 24 | 25 | externalip "github.com/Mirantis/k8s-externalipcontroller/pkg" 26 | ) 27 | 28 | func init() { 29 | Root.AddCommand(Naive) 30 | } 31 | 32 | var Naive = &cobra.Command{ 33 | Aliases: []string{"n"}, 34 | Use: "naivecontroller", 35 | RunE: func(cmd *cobra.Command, args []string) error { 36 | return InitNaiveController() 37 | }, 38 | } 39 | 40 | func InitNaiveController() error { 41 | kubeconfig := AppOpts.Kubeconfig 42 | iface := AppOpts.Iface 43 | mask := AppOpts.Mask 44 | 45 | glog.V(4).Infof("Starting external ip controller using link: %s and mask: /%s", iface, mask) 46 | stopCh := make(chan struct{}) 47 | 48 | var err error 49 | var config *rest.Config 50 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 51 | if err != nil { 52 | glog.Errorf("Error parsing config. %v", err) 53 | return err 54 | } 55 | 56 | host, err := os.Hostname() 57 | if err != nil { 58 | return err 59 | } 60 | 61 | c, err := externalip.NewExternalIpController(config, host, iface, mask, AppOpts.ResyncInterval) 62 | if err != nil { 63 | return err 64 | } 65 | c.Run(stopCh) 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /doc/simple-deployment-scheme.md: -------------------------------------------------------------------------------- 1 | Simple Deployment Scheme 2 | ======================== 3 | 4 | ## Introduction 5 | 6 | One of the possible ways to expose k8s services on a bare metal deployments is 7 | using External IPs. Each node runs a kube-proxy process which programs `iptables` 8 | rules to trap access to External IPs and redirect them to the correct backends. 9 | 10 | So in order to access k8s service from the outside we just need to route public 11 | traffic to one of k8s worker nodes which has `kube-proxy` running and thus has 12 | needed `iptables` rules for External IPs. 13 | 14 | ## Proposed solution 15 | 16 | External IP controller is k8s application which is deployed on top of k8s 17 | cluster and which configures External IPs on k8s worker node(s) to provide 18 | IP connectivity. 19 | 20 | This document describes a simple deployment scheme for External IP controller. 21 | 22 | We assume that: 23 | * Public network (network for External IPs) is routable to nodes. 24 | * We want to expose services to external clients via external IPs. 25 | 26 | ## Deployment scheme 27 | 28 | ![Deployment scheme](images/simple-scheme.png) 29 | 30 | Description: 31 | * External IP controller kubernetes application is running on one of the nodes 32 | (`replicas=1`). 33 | * On start it pulls information about services from `kube-api` and brings up 34 | all External IPs on the specified interface (eth0 in our example above). 35 | * It watches `kube-api` for updates in services with External IPs and: 36 | * When new External IPs appear it brings them up. 37 | * When service is removed it removes appropriate External IPs from the 38 | interface. 39 | * Kubernetes provides fail-over for External IP controller. Since we have 40 | `replicas` set to 1, then we'll have only one instance running in a cluster to 41 | avoid IPs duplication. And when there's a problem with k8s node, External IP 42 | controller will be spawned on a new k8s worker node and bring External IPs up 43 | on that node. 44 | -------------------------------------------------------------------------------- /cmd/app/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "os" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/claimcontroller" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 23 | 24 | "github.com/spf13/cobra" 25 | "k8s.io/client-go/1.5/rest" 26 | "k8s.io/client-go/1.5/tools/clientcmd" 27 | ) 28 | 29 | func init() { 30 | Root.AddCommand(Controller) 31 | } 32 | 33 | var Controller = &cobra.Command{ 34 | Aliases: []string{"c"}, 35 | Use: "claimcontroller", 36 | RunE: func(cmd *cobra.Command, args []string) error { 37 | return InitController() 38 | }, 39 | } 40 | 41 | func InitController() error { 42 | var err error 43 | var config *rest.Config 44 | kubeconfig := AppOpts.Kubeconfig 45 | iface := AppOpts.Iface 46 | hostname := AppOpts.Hostname 47 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 48 | if err != nil { 49 | return err 50 | } 51 | uid := hostname 52 | if hostname == "" { 53 | uid, err = os.Hostname() 54 | } 55 | if err != nil { 56 | return err 57 | } 58 | stop := make(chan struct{}) 59 | c, err := claimcontroller.NewClaimController(iface, uid, config, AppOpts.ResyncInterval, AppOpts.HeartbeatInterval) 60 | if err != nil { 61 | return err 62 | } 63 | err = extensions.EnsureThirdPartyResourcesExist(c.Clientset) 64 | if err != nil { 65 | return err 66 | } 67 | err = extensions.WaitThirdPartyResources(c.ExtensionsClientset, 10*time.Second, 1*time.Second) 68 | if err != nil { 69 | return err 70 | } 71 | c.Run(stop) 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/extensions/register.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "strings" 19 | "time" 20 | 21 | "k8s.io/client-go/1.5/kubernetes" 22 | "k8s.io/client-go/1.5/pkg/api" 23 | "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1" 24 | ) 25 | 26 | func EnsureThirdPartyResourcesExist(ki kubernetes.Interface) error { 27 | resourceNames := []string{"ip-node", "ip-claim", "ip-claim-pool"} 28 | for _, resName := range resourceNames { 29 | if err := ensureThirdPartyResource(ki, resName); err != nil { 30 | return err 31 | } 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func RemoveThirdPartyResources(ki kubernetes.Interface) { 38 | for _, name := range []string{"ip-node", "ip-claim", "ip-claim-pool"} { 39 | fullName := strings.Join([]string{name, GroupName}, ".") 40 | ki.Extensions().ThirdPartyResources().Delete(fullName, &api.DeleteOptions{}) 41 | } 42 | } 43 | 44 | func ensureThirdPartyResource(ki kubernetes.Interface, name string) error { 45 | fullName := strings.Join([]string{name, GroupName}, ".") 46 | _, err := ki.Extensions().ThirdPartyResources().Get(fullName) 47 | if err == nil { 48 | return nil 49 | } 50 | 51 | resource := &v1beta1.ThirdPartyResource{ 52 | Versions: []v1beta1.APIVersion{ 53 | {Name: Version}, 54 | }} 55 | resource.SetName(fullName) 56 | _, err = ki.Extensions().ThirdPartyResources().Create(resource) 57 | return err 58 | } 59 | 60 | func WaitThirdPartyResources(ext ExtensionsClientset, timeout time.Duration, interval time.Duration) (err error) { 61 | timeoutChan := time.After(timeout) 62 | intervalChan := time.Tick(interval) 63 | for { 64 | select { 65 | case <-timeoutChan: 66 | return err 67 | case <-intervalChan: 68 | _, err = ext.IPClaims().List(api.ListOptions{}) 69 | if err != nil { 70 | continue 71 | } 72 | _, err = ext.IPNodes().List(api.ListOptions{}) 73 | if err != nil { 74 | continue 75 | } 76 | _, err = ext.IPClaimPools().List(api.ListOptions{}) 77 | if err != nil { 78 | continue 79 | } 80 | return nil 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /cmd/app/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package app 15 | 16 | import ( 17 | "fmt" 18 | "errors" 19 | "strings" 20 | "time" 21 | 22 | "k8s.io/kubernetes/pkg/apis/componentconfig" 23 | "k8s.io/kubernetes/pkg/client/leaderelection" 24 | 25 | "github.com/spf13/pflag" 26 | ) 27 | 28 | type options struct { 29 | Hostname string 30 | Iface string 31 | Kubeconfig string 32 | Mask string 33 | NodeFilter string 34 | 35 | HeartbeatInterval time.Duration 36 | MonitorInterval time.Duration 37 | ResyncInterval time.Duration 38 | 39 | LeaderElection componentconfig.LeaderElectionConfiguration 40 | } 41 | 42 | var AppOpts = options{} 43 | 44 | var NodeFilters = []string{ 45 | "fair", 46 | "first-alive", 47 | } 48 | 49 | func init() { 50 | AppOpts.AddFlags(pflag.CommandLine) 51 | } 52 | 53 | func (o *options) AddFlags(fs *pflag.FlagSet) { 54 | fs.StringVar(&o.Iface, "iface", "eth0", "Current interface will be used to assign ip addresses") 55 | fs.StringVar(&o.Mask, "mask", "32", "mask part of the cidr") 56 | fs.StringVar(&o.Kubeconfig, "kubeconfig", "", "kubeconfig to use with kubernetes client") 57 | fs.StringVar(&o.Hostname, "hostname", "", "We will use os.Hostname if none provided") 58 | filterList := strings.Join(NodeFilters, "|") 59 | fs.StringVar(&o.NodeFilter, "nodefilter", NodeFilters[0], fmt.Sprintf("Possible values: %s. We will use '%s' if none was provided.", filterList, NodeFilters[0])) 60 | fs.DurationVar(&o.ResyncInterval, "resync", 20*time.Second, "Time to resync state for all ips") 61 | fs.DurationVar(&o.HeartbeatInterval, "hb", 2*time.Second, "How often to send heartbeats from controllers?") 62 | fs.DurationVar(&o.MonitorInterval, "monitor", 4*time.Second, "How often to check controllers liveness?") 63 | o.LeaderElection = leaderelection.DefaultLeaderElectionConfiguration() 64 | leaderelection.BindFlags(&o.LeaderElection, fs) 65 | } 66 | 67 | func (o *options) CheckFlags() error { 68 | for _, f := range NodeFilters { 69 | if o.NodeFilter == f { 70 | return nil 71 | } 72 | } 73 | return errors.New("Incorrect node filter is provided") 74 | } -------------------------------------------------------------------------------- /pkg/workqueue/workqueue.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package workqueue 16 | 17 | import ( 18 | "sync" 19 | ) 20 | 21 | type QueueAddType interface { 22 | Add(interface{}) 23 | } 24 | 25 | type QueueType interface { 26 | QueueAddType 27 | Get() (interface{}, bool) 28 | Done(interface{}) 29 | Remove(interface{}) 30 | Close() 31 | Len() int 32 | } 33 | 34 | func NewQueue() *Queue { 35 | return &Queue{ 36 | cond: sync.NewCond(&sync.Mutex{}), 37 | added: map[interface{}]bool{}, 38 | processing: map[interface{}]bool{}, 39 | queue: []interface{}{}, 40 | } 41 | } 42 | 43 | type Queue struct { 44 | cond *sync.Cond 45 | added map[interface{}]bool 46 | processing map[interface{}]bool 47 | closed bool 48 | queue []interface{} 49 | } 50 | 51 | func (n *Queue) Add(item interface{}) { 52 | n.cond.L.Lock() 53 | defer n.cond.L.Unlock() 54 | if n.closed { 55 | return 56 | } 57 | if _, exists := n.added[item]; exists { 58 | return 59 | } 60 | n.added[item] = true 61 | if _, exists := n.processing[item]; exists { 62 | return 63 | } 64 | n.queue = append(n.queue, item) 65 | n.cond.Signal() 66 | } 67 | 68 | func (n *Queue) Len() int { 69 | return len(n.queue) 70 | } 71 | 72 | func (n *Queue) Close() { 73 | n.cond.L.Lock() 74 | defer n.cond.L.Unlock() 75 | n.closed = true 76 | n.cond.Broadcast() 77 | } 78 | 79 | func (n *Queue) Get() (item interface{}, quit bool) { 80 | n.cond.L.Lock() 81 | defer n.cond.L.Unlock() 82 | 83 | if len(n.queue) == 0 && !n.closed { 84 | n.cond.Wait() 85 | } 86 | 87 | if len(n.queue) == 0 { 88 | return nil, n.closed 89 | } 90 | 91 | for { 92 | item, n.queue = n.queue[0], n.queue[1:] 93 | // item was removed and shouldn't be processed 94 | if _, exists := n.added[item]; !exists { 95 | if len(n.queue) == 0 { 96 | return nil, n.closed 97 | } 98 | continue 99 | } 100 | break 101 | } 102 | n.processing[item] = true 103 | delete(n.added, item) 104 | return item, false 105 | } 106 | 107 | func (n *Queue) Done(item interface{}) { 108 | n.cond.L.Lock() 109 | defer n.cond.L.Unlock() 110 | 111 | delete(n.processing, item) 112 | 113 | if _, exists := n.added[item]; exists { 114 | n.queue = append(n.queue, item) 115 | n.cond.Signal() 116 | } 117 | } 118 | 119 | // Remove will prevent item from being processed 120 | func (n *Queue) Remove(item interface{}) { 121 | n.cond.L.Lock() 122 | defer n.cond.L.Unlock() 123 | 124 | if _, exists := n.added[item]; exists { 125 | delete(n.added, item) 126 | } 127 | } 128 | 129 | type ProcessType interface { 130 | Process(func(item interface{}) error) 131 | } 132 | 133 | type ProcessingQueue struct { 134 | *Queue 135 | } 136 | 137 | func (p *ProcessingQueue) Process(f func(item interface{}) error) error { 138 | item, _ := p.Get() 139 | defer p.Done(item) 140 | if err := f(item); err != nil { 141 | p.Add(item) 142 | return err 143 | } 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /doc/fail-over-optimization.md: -------------------------------------------------------------------------------- 1 | Fail-Over Optimization 2 | ====================== 3 | 4 | External IP fail-over functionality depends on both External IP Controller 5 | application capabilities and a number of k8s parameters. 6 | Application itself provides fail-over capabilities in claims mode. 7 | There is a set of k8s parameters that affect pods respawning lag after some 8 | node goes offline. These parameters have different effect on application 9 | fail-over functionality depending on application operating mode. 10 | 11 | ## Pods Respawning and Corresponding k8s Parameters 12 | 13 | Kubernetes cares only about pods availability including pods where External IP 14 | Controller application is running. So, when some node goes offline k8s detects 15 | that and respawns the corresponding pods on some another node. 16 | 17 | The following parameters affect pods respawning lag: 18 | * `--node-status-update-frequency` - kubelet's parameter, 10s by default; 19 | * `--node-monitor-period` - parameter of kube-controller-manager, 5s by default; 20 | * `--node-monitor-grace-period` - parameter of kube-controller-manager, 40s by 21 | default; 22 | * `--pod-eviction-timeout` - parameter of kube-controller-manager, 5m by default; 23 | Please refer to kubernetes docs for more detail. In short, `node-monitor-grace-period` 24 | determines how fast node will be considered as being offline, 25 | `node-status-update-frequency` and `node-monitor-period` should be proportionally 26 | smaller than that. `pod-eviction-timeout` determines when pods will be 27 | respawned on some other node after failed node status will be changed to `NotReady`. 28 | Changing default values to `--node-status-update-frequency=4s`, 29 | `--node-monitor-period=2s`, `--node-monitor-grace-period=16s`, 30 | `--pod-eviction-timeout=20s` allows to decrease pods respawning lag from 5 minutes 31 | to about 20 seconds. These values were tested on a lab with 5-6 nodes clusters. 32 | 33 | ## Simple Mode 34 | 35 | In simple mode, kubernetes will respawn IP Controller pods so that application 36 | is able to continue operating properly (will respawn corresponding IPs on a new 37 | node). 38 | As there is the only controller part which is running in simple mode, so it is 39 | just kubernetes who cares about controllers' availability. 40 | 41 | ## Claims Mode 42 | 43 | In claims mode, kubernetes will respawn IP Scheduler pods and IP Controller 44 | pods. But in claims mode it is not so critical for the application as leader 45 | election support for schedulers and monitoring of controllers reachability 46 | improves availability of the application. 47 | Thanks to the leader election, one scheduler will be active at any 48 | particular time. If the active scheduler becomes unavailable, then 49 | another instance of scheduler becomes active. So, scheduler remains functional 50 | while at least one operational node has IP Scheduler pods. Scheduler module 51 | monitors controllers availability and reschedules IP claims to healthy 52 | controllers from unavailable ones. 53 | In overall, application will continue to work properly while at least one 54 | scheduler and one controller remain operational. 55 | There is a set of application parameters that affect IP fail-over: 56 | `hb`, `leader-elect` and `monitor`. For parameters description, please refer to 57 | "Basic Modules and Operating Modes" document for more detail. 58 | 59 | ## Known Issues 60 | 61 | The following issues with kubernetes were discovered when any of master or etcd 62 | nodes goes down. 63 | Regardless of the settings described above there are chances that pods from 64 | failed nodes will be respawned with about 15-20 minutes lag. 65 | Another issue is that kube-proxy does not resetup routes so applications cannot 66 | access kube-API when some of the master nodes go down. 67 | This behaviour was observed with k8s v.1.5.x and calico v1.1.0-rc3, v1.1.0-rc5 68 | within 25-30% cases. 69 | -------------------------------------------------------------------------------- /pkg/controller_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package externalip 16 | 17 | import ( 18 | "strings" 19 | "testing" 20 | "time" 21 | 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 23 | "github.com/stretchr/testify/mock" 24 | 25 | "k8s.io/client-go/1.5/pkg/api/v1" 26 | "k8s.io/client-go/1.5/tools/cache" 27 | fcache "k8s.io/client-go/1.5/tools/cache/testing" 28 | ) 29 | 30 | type fakeIpHandler struct { 31 | mock.Mock 32 | syncer chan struct{} 33 | } 34 | 35 | func (f *fakeIpHandler) Add(iface, cidr string) error { 36 | args := f.Called(iface, cidr) 37 | f.syncer <- struct{}{} 38 | return args.Error(0) 39 | } 40 | 41 | func (f *fakeIpHandler) Del(iface, cidr string) error { 42 | args := f.Called(iface, cidr) 43 | f.syncer <- struct{}{} 44 | return args.Error(0) 45 | } 46 | 47 | func TestControllerServicesAdded(t *testing.T) { 48 | t.Log("started assign ip test") 49 | source := fcache.NewFakeControllerSource() 50 | syncer := make(chan struct{}, 6) 51 | fake := &fakeIpHandler{syncer: syncer} 52 | c := &ExternalIpController{ 53 | Iface: "eth0", 54 | Mask: "24", 55 | source: source, 56 | ipHandler: fake, 57 | Queue: workqueue.NewQueue(), 58 | } 59 | 60 | stopCh := make(chan struct{}) 61 | defer close(stopCh) 62 | go c.Run(stopCh) 63 | 64 | testIps := [][]string{ 65 | {"10.10.0.2", "10.10.0.3"}, 66 | {"10.10.0.2", "10.10.0.3", "10.10.0.4"}, 67 | {"10.10.0.5"}, 68 | } 69 | 70 | added := make(map[string]bool) 71 | for i, ips := range testIps { 72 | for _, ip := range ips { 73 | if _, present := added[ip]; !present { 74 | fake.On("Add", c.Iface, strings.Join([]string{ip, c.Mask}, "/")).Return(nil) 75 | added[ip] = true 76 | } 77 | } 78 | source.Add(&v1.Service{ 79 | ObjectMeta: v1.ObjectMeta{Name: "service-" + string(i)}, 80 | Spec: v1.ServiceSpec{ExternalIPs: ips}, 81 | }) 82 | } 83 | 84 | for i := 0; i < len(added); i++ { 85 | select { 86 | case <-time.After(200 * time.Millisecond): 87 | t.Errorf("Waiting for calls failed. Current calls %v", fake.Calls) 88 | case <-fake.syncer: 89 | } 90 | } 91 | } 92 | 93 | func TestProcessExternalIps(t *testing.T) { 94 | fake := &fakeIpHandler{syncer: make(chan struct{}, 6)} 95 | c := &ExternalIpController{ 96 | Iface: "eth0", 97 | Mask: "24", 98 | ipHandler: fake, 99 | Queue: workqueue.NewQueue(), 100 | } 101 | testIps := [][]string{ 102 | {"10.10.0.2", "10.10.0.3"}, 103 | {"10.10.0.2", "10.10.0.3", "10.10.0.4"}, 104 | {"10.10.0.5"}, 105 | } 106 | go c.worker() 107 | 108 | added := make(map[string]bool) 109 | for _, ips := range testIps { 110 | for _, ip := range ips { 111 | if _, present := added[ip]; !present { 112 | fake.On("Add", c.Iface, strings.Join([]string{ip, c.Mask}, "/")).Return(nil) 113 | added[ip] = true 114 | } 115 | } 116 | c.processServiceExternalIPs(nil, &v1.Service{Spec: v1.ServiceSpec{ExternalIPs: ips}}, cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)) 117 | } 118 | 119 | for i := 0; i < len(added); i++ { 120 | select { 121 | case <-time.After(200 * time.Millisecond): 122 | t.Errorf("Waiting for calls failed. Current calls %v", fake.Calls) 123 | case <-fake.syncer: 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /pkg/extensions/types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func checkFreeIP(t *testing.T, p *IpClaimPool, expected string) { 22 | actual, err := p.AvailableIP() 23 | if err != nil { 24 | t.Errorf("Error must not occur during AvailableIP() method; details --> %v\n", err) 25 | } 26 | 27 | if expected != actual { 28 | t.Errorf("Actual free IP %v is not as expected %v\n", actual, expected) 29 | } 30 | } 31 | 32 | func checkNoFreeIPError(t *testing.T, p *IpClaimPool) { 33 | ip, err := p.AvailableIP() 34 | 35 | if len(ip) != 0 { 36 | t.Error("AvailableIP must return empty string as IP value in case there is no free IP") 37 | } 38 | if err == nil { 39 | t.Error("AvailableIP must return error in case there is no free IP") 40 | } 41 | expectedErrMsg := "There is no free IP left in the pool" 42 | if err.Error() != expectedErrMsg { 43 | t.Errorf("Message of the returned by AvailableIP error is not as expected ('%v'). Actual: %v", expectedErrMsg, err) 44 | } 45 | } 46 | 47 | func TestIpClaimPoolAvailableIPFromCIDR(t *testing.T) { 48 | cidr := "192.168.16.248/29" 49 | allocated := map[string]string{ 50 | "192.168.16.249": "test-claim-249", 51 | "192.168.16.250": "test-claim-250", 52 | "192.168.16.252": "test-claim-252", 53 | "192.168.16.253": "test-claim-253", 54 | } 55 | expectedIP := "192.168.16.251" 56 | 57 | ClaimPool := &IpClaimPool{ 58 | Spec: IpClaimPoolSpec{ 59 | CIDR: cidr, 60 | Ranges: nil, 61 | Allocated: allocated, 62 | }, 63 | } 64 | 65 | checkFreeIP(t, ClaimPool, expectedIP) 66 | 67 | allocated["192.168.16.254"] = "test-claim-254" 68 | allocated["192.168.16.251"] = "test-claim-251" 69 | 70 | checkNoFreeIPError(t, ClaimPool) 71 | } 72 | 73 | func TestIpClaimPoolAvailableIPFromRange(t *testing.T) { 74 | cidr := "192.168.16.248/29" 75 | IPranges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 76 | 77 | allocated := map[string]string{ 78 | "192.168.16.250": "test-claim-250", 79 | "192.168.16.251": "test-claim-251", 80 | } 81 | 82 | expectedIP := "192.168.16.252" 83 | 84 | ClaimPool := &IpClaimPool{ 85 | Spec: IpClaimPoolSpec{ 86 | CIDR: cidr, 87 | Ranges: IPranges, 88 | Allocated: allocated, 89 | }, 90 | } 91 | 92 | checkFreeIP(t, ClaimPool, expectedIP) 93 | 94 | allocated["192.168.16.252"] = "test-claim-252" 95 | 96 | checkNoFreeIPError(t, ClaimPool) 97 | } 98 | 99 | func TestIpClaimPoolRangesProperlyProcessed(t *testing.T) { 100 | cidr := "192.168.16.248/29" 101 | IPranges := [][]string{ 102 | []string{"192.168.16.249", "192.168.16.250"}, 103 | []string{"192.168.16.252", "192.168.16.253"}, 104 | } 105 | 106 | allocated := map[string]string{ 107 | "192.168.16.249": "test-claim-249", 108 | "192.168.16.250": "test-claim-250", 109 | "192.168.16.252": "test-claim-252", 110 | } 111 | 112 | expectedIP := "192.168.16.253" 113 | 114 | ClaimPool := &IpClaimPool{ 115 | Spec: IpClaimPoolSpec{ 116 | CIDR: cidr, 117 | Ranges: IPranges, 118 | Allocated: allocated, 119 | }, 120 | } 121 | 122 | checkFreeIP(t, ClaimPool, expectedIP) 123 | 124 | allocated["192.168.16.253"] = "test-claim-253" 125 | 126 | checkNoFreeIPError(t, ClaimPool) 127 | } 128 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | IMAGE_REPO ?= mirantis/k8s-externalipcontroller 2 | IMAGE_TAG ?= latest 3 | DOCKER_BUILD ?= no 4 | 5 | BUILD_DIR = _output 6 | VENDOR_DIR = vendor 7 | ROOT_DIR = $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) 8 | 9 | ENV_PREPARE_MARKER = .env-prepare.complete 10 | BUILD_IMAGE_MARKER = .build-image.complete 11 | 12 | ifeq ($(DOCKER_BUILD), yes) 13 | _DOCKER_GOPATH = /go 14 | _DOCKER_WORKDIR = $(_DOCKER_GOPATH)/src/github.com/Mirantis/k8s-externalipcontroller/ 15 | _DOCKER_IMAGE = golang:1.7 16 | DOCKER_DEPS = apt-get update; apt-get install -y libpcap-dev; 17 | DOCKER_EXEC = docker run --rm -it -v "$(ROOT_DIR):$(_DOCKER_WORKDIR)" \ 18 | -w "$(_DOCKER_WORKDIR)" $(_DOCKER_IMAGE) 19 | else 20 | DOCKER_EXEC = 21 | DOCKER_DEPS = 22 | endif 23 | 24 | .PHONY: help 25 | help: 26 | @echo "Usage: 'make '" 27 | @echo "" 28 | @echo "Targets:" 29 | @echo "help - Print this message and exit" 30 | @echo "get-deps - Install project dependencies" 31 | @echo "containerized-build - Build ipmanager binary in container" 32 | @echo "build - Build ipmanager binary" 33 | @echo "build-image - Build docker image" 34 | @echo "test - Run all tests" 35 | @echo "unit - Run unit tests" 36 | @echo "integration - Run integration tests" 37 | @echo "e2e - Run e2e tests" 38 | @echo "clean - Delete binaries" 39 | @echo "clean-all - Delete binaries and vendor files" 40 | 41 | .PHONY: get-deps 42 | get-deps: $(VENDOR_DIR) 43 | 44 | 45 | .PHONY: build 46 | build: $(BUILD_DIR)/ipmanager 47 | 48 | 49 | .PHONY: containerized-build 50 | containerized-build: 51 | make build DOCKER_BUILD=yes 52 | 53 | 54 | .PHONY: build-image 55 | build-image: $(BUILD_IMAGE_MARKER) 56 | 57 | 58 | .PHONY: unit 59 | unit: 60 | $(DOCKER_EXEC) go test -v ./pkg/... 61 | 62 | 63 | .PHONY: integration 64 | integration: $(BUILD_DIR)/integration.test $(ENV_PREPARE_MARKER) 65 | sudo $(BUILD_DIR)/integration.test --ginkgo.v --logtostderr --v=10 66 | 67 | 68 | .PHONY: e2e 69 | e2e: $(BUILD_DIR)/e2e.test $(ENV_PREPARE_MARKER) 70 | sudo $(BUILD_DIR)/e2e.test --master=http://localhost:8888 --testlink=docker0 -ginkgo.v 71 | 72 | 73 | .PHONY: test 74 | test: unit integration e2e 75 | 76 | 77 | .PHONY: clean 78 | clean: 79 | rm -rf $(BUILD_DIR) 80 | 81 | 82 | .PHONY: clean-all 83 | clean-all: clean 84 | rm -rf $(VENDOR_DIR) 85 | rm -f $(BUILD_IMAGE_MARKER) 86 | docker rmi -f $(IMAGE_REPO):$(IMAGE_TAG) 87 | 88 | 89 | $(BUILD_DIR): 90 | mkdir -p $(BUILD_DIR) 91 | 92 | 93 | $(BUILD_DIR)/ipmanager: $(BUILD_DIR) $(VENDOR_DIR) 94 | $(DOCKER_EXEC) bash -xc '$(DOCKER_DEPS) \ 95 | go build --ldflags "-extldflags \"-static\"" \ 96 | -o $@ ./cmd/ipmanager/ ; \ 97 | chown $(shell id -u):$(shell id -g) -R _output' 98 | 99 | 100 | $(BUILD_DIR)/e2e.test: $(BUILD_DIR) $(VENDOR_DIR) 101 | $(DOCKER_EXEC) go test -c -o $@ ./test/e2e/ 102 | 103 | 104 | $(BUILD_DIR)/integration.test: $(BUILD_DIR) $(VENDOR_DIR) 105 | $(DOCKER_EXEC) go test -c -o $@ ./test/integration/ 106 | 107 | 108 | $(BUILD_IMAGE_MARKER): $(BUILD_DIR)/ipmanager 109 | docker build -t $(IMAGE_REPO):$(IMAGE_TAG) . 110 | echo > $(BUILD_IMAGE_MARKER) 111 | 112 | 113 | $(VENDOR_DIR): 114 | $(DOCKER_EXEC) bash -xc 'go get github.com/Masterminds/glide && \ 115 | glide install --strip-vendor; \ 116 | chown $(shell id -u):$(shell id -g) -R vendor' 117 | 118 | 119 | $(ENV_PREPARE_MARKER): build-image 120 | ./scripts/kube.sh 121 | ./scripts/dind.sh 122 | CONTAINER_ID=$$(docker create $(IMAGE_REPO):$(IMAGE_TAG) bash) && \ 123 | docker export $$CONTAINER_ID > $(BUILD_DIR)/ipcontroller.tar 124 | docker cp $(BUILD_DIR)/ipcontroller.tar dind_node_1:/tmp 125 | docker exec -ti dind_node_1 docker import /tmp/ipcontroller.tar $(IMAGE_REPO):$(IMAGE_TAG) 126 | docker cp $(BUILD_DIR)/ipcontroller.tar dind_node_2:/tmp 127 | docker exec -ti dind_node_2 docker import /tmp/ipcontroller.tar $(IMAGE_REPO):$(IMAGE_TAG) 128 | echo > $(ENV_PREPARE_MARKER) 129 | -------------------------------------------------------------------------------- /cmd/app/scheduler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "os" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/scheduler" 23 | 24 | "github.com/golang/glog" 25 | "github.com/spf13/cobra" 26 | "k8s.io/client-go/1.5/rest" 27 | "k8s.io/client-go/1.5/tools/clientcmd" 28 | "k8s.io/kubernetes/pkg/api" 29 | clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" 30 | "k8s.io/kubernetes/pkg/client/leaderelection" 31 | "k8s.io/kubernetes/pkg/client/leaderelection/resourcelock" 32 | "k8s.io/kubernetes/pkg/client/record" 33 | "k8s.io/kubernetes/pkg/client/restclient" 34 | ) 35 | 36 | func init() { 37 | Root.AddCommand(Scheduler) 38 | } 39 | 40 | var Scheduler = &cobra.Command{ 41 | Aliases: []string{"s"}, 42 | Use: "scheduler", 43 | RunE: func(cmd *cobra.Command, args []string) error { 44 | return InitScheduler() 45 | }, 46 | } 47 | 48 | func InitScheduler() error { 49 | var err error 50 | var config *rest.Config 51 | kubeconfig := AppOpts.Kubeconfig 52 | mask := AppOpts.Mask 53 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 54 | if err != nil { 55 | glog.Errorf("Error parsing config. %v", err) 56 | os.Exit(1) 57 | } 58 | stop := make(chan struct{}) 59 | s, err := scheduler.NewIPClaimScheduler(config, mask, AppOpts.MonitorInterval, AppOpts.NodeFilter) 60 | if err != nil { 61 | glog.Errorf("Crashed during scheduler initialization: %v", err) 62 | os.Exit(2) 63 | } 64 | err = extensions.EnsureThirdPartyResourcesExist(s.Clientset) 65 | if err != nil { 66 | glog.Fatalf("Crashed while initializing third party resources: %v", err) 67 | } 68 | err = extensions.WaitThirdPartyResources(s.ExtensionsClientset, 10*time.Second, 1*time.Second) 69 | if err != nil { 70 | glog.Fatalf("URLs for tprs are not registered: %v", err) 71 | } 72 | 73 | if !AppOpts.LeaderElection.LeaderElect { 74 | s.Run(stop) 75 | os.Exit(0) 76 | } 77 | glog.V(0).Infof("Running with leader election turned on.") 78 | run := func(_ <-chan struct{}) { 79 | s.Run(stop) 80 | } 81 | 82 | id, err := os.Hostname() 83 | if err != nil { 84 | glog.Fatalf("Cannot get hostname %v", err) 85 | } 86 | 87 | glog.V(0).Infof("Starting scheduler in leader election mode with id=%v", id) 88 | kubernetesConfig, err := restclient.InClusterConfig() 89 | if err != nil { 90 | glog.Fatalf("Error creating config %v", err) 91 | } 92 | leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubernetesConfig, "leader-election")) 93 | if err != nil { 94 | glog.Fatalf("Incorrect configuration %v", err) 95 | } 96 | eventBroadcaster := record.NewBroadcaster() 97 | recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "ipclaim-scheduler"}) 98 | 99 | rl := resourcelock.EndpointsLock{ 100 | EndpointsMeta: api.ObjectMeta{ 101 | Namespace: "kube-system", 102 | Name: "ipclaim-scheduler", 103 | }, 104 | Client: leaderElectionClient, 105 | LockConfig: resourcelock.ResourceLockConfig{ 106 | Identity: id, 107 | EventRecorder: recorder, 108 | }, 109 | } 110 | 111 | leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ 112 | Lock: &rl, 113 | LeaseDuration: AppOpts.LeaderElection.LeaseDuration.Duration, 114 | RenewDeadline: AppOpts.LeaderElection.RenewDeadline.Duration, 115 | RetryPeriod: AppOpts.LeaderElection.RetryPeriod.Duration, 116 | Callbacks: leaderelection.LeaderCallbacks{ 117 | OnStartedLeading: run, 118 | OnStoppedLeading: func() { 119 | glog.Fatalf("lost master") 120 | }, 121 | }, 122 | }) 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /pkg/claimcontroller/controller_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package claimcontroller 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | fclient "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions/testing" 23 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 24 | 25 | "github.com/Mirantis/k8s-externalipcontroller/pkg/utils" 26 | "github.com/stretchr/testify/assert" 27 | "github.com/stretchr/testify/mock" 28 | "k8s.io/client-go/1.5/pkg/api" 29 | "k8s.io/client-go/1.5/pkg/api/errors" 30 | "k8s.io/client-go/1.5/pkg/api/unversioned" 31 | fcache "k8s.io/client-go/1.5/tools/cache/testing" 32 | ) 33 | 34 | type fakeIpHandler struct { 35 | mock.Mock 36 | } 37 | 38 | func (f *fakeIpHandler) Add(iface, cidr string) error { 39 | args := f.Called(iface, cidr) 40 | return args.Error(0) 41 | } 42 | 43 | func (f *fakeIpHandler) Del(iface, cidr string) error { 44 | args := f.Called(iface, cidr) 45 | return args.Error(0) 46 | } 47 | 48 | func TestClaimWatcher(t *testing.T) { 49 | ext := fclient.NewFakeExtClientset() 50 | lw := fcache.NewFakeControllerSource() 51 | queue := workqueue.NewQueue() 52 | fiphandler := &fakeIpHandler{} 53 | defer queue.Close() 54 | stop := make(chan struct{}) 55 | defer close(stop) 56 | c := claimController{ 57 | Uid: "first", 58 | Iface: "eth0", 59 | ExtensionsClientset: ext, 60 | claimSource: lw, 61 | queue: queue, 62 | iphandler: fiphandler, 63 | } 64 | go c.claimWatcher(stop) 65 | go c.worker() 66 | claim := &extensions.IpClaim{ 67 | Metadata: api.ObjectMeta{Name: "10.10.0.2-24"}, 68 | Spec: extensions.IpClaimSpec{Cidr: "10.10.0.2/24", NodeName: "first"}, 69 | } 70 | fiphandler.On("Add", c.Iface, claim.Spec.Cidr).Return(nil) 71 | lw.Add(claim) 72 | utils.EventualCondition(t, time.Second*1, func() bool { 73 | return assert.ObjectsAreEqual(1, len(fiphandler.Calls)) 74 | }, "Unexpect calls to iphandler", fiphandler.Calls) 75 | assert.Equal(t, fiphandler.Calls[0].Arguments[0].(string), c.Iface, "Unexpected interface passed to netutils") 76 | assert.Equal(t, fiphandler.Calls[0].Arguments[1].(string), claim.Spec.Cidr, "Unexpected cidr") 77 | 78 | lw.Delete(claim) 79 | fiphandler.On("Del", c.Iface, claim.Spec.Cidr).Return(nil) 80 | utils.EventualCondition(t, time.Second*1, func() bool { 81 | return assert.ObjectsAreEqual(2, len(fiphandler.Calls)) 82 | }, "Unexpect calls to iphandler", fiphandler.Calls) 83 | assert.Equal(t, fiphandler.Calls[1].Arguments[0].(string), c.Iface, "Unexpected interface passed to netutils") 84 | assert.Equal(t, fiphandler.Calls[1].Arguments[1].(string), claim.Spec.Cidr, "Unexpected cidr") 85 | } 86 | 87 | func TestHeartbeatIpNode(t *testing.T) { 88 | ext := fclient.NewFakeExtClientset() 89 | ticker := make(chan time.Time, 3) 90 | for i := 0; i < 3; i++ { 91 | ticker <- time.Time{} 92 | } 93 | stop := make(chan struct{}) 94 | c := claimController{ 95 | Uid: "first", 96 | ExtensionsClientset: ext, 97 | } 98 | qualResource := unversioned.GroupResource{ 99 | Group: "ipcontroller", 100 | Resource: "ipnode", 101 | } 102 | ipnode := &extensions.IpNode{ 103 | Metadata: api.ObjectMeta{Name: c.Uid}, 104 | } 105 | ext.Ipnodes.On("Get", c.Uid).Return(&extensions.IpNode{}, errors.NewNotFound(qualResource, c.Uid)) 106 | ext.Ipnodes.On("Create", mock.Anything).Return(nil) 107 | ext.Ipnodes.On("Get", c.Uid).Return(ipnode, nil).Twice() 108 | ext.Ipnodes.On("Update", mock.Anything).Return(nil).Twice() 109 | go c.heartbeatIpNode(stop, ticker) 110 | utils.EventualCondition(t, time.Second*1, func() bool { 111 | return assert.ObjectsAreEqual(6, len(ext.Ipnodes.Calls)) 112 | }, "Unexpect calls to iphandler", ext.Ipnodes.Calls) 113 | } 114 | -------------------------------------------------------------------------------- /pkg/netutils/netutils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package netutils 16 | 17 | import ( 18 | "net" 19 | 20 | "github.com/golang/glog" 21 | "github.com/vishvananda/netlink" 22 | "github.com/google/gopacket" 23 | "github.com/google/gopacket/layers" 24 | "github.com/google/gopacket/pcap" 25 | ) 26 | 27 | func writeARP(handle *pcap.Handle, iface *net.Interface, addr *net.IPNet) error { 28 | // Set up all the layers' fields we can. 29 | eth := layers.Ethernet{ 30 | SrcMAC: iface.HardwareAddr, 31 | DstMAC: net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, 32 | EthernetType: layers.EthernetTypeARP, 33 | } 34 | arp := layers.ARP{ 35 | AddrType: layers.LinkTypeEthernet, 36 | Protocol: layers.EthernetTypeIPv4, 37 | HwAddressSize: 6, 38 | ProtAddressSize: 4, 39 | Operation: layers.ARPRequest, 40 | SourceHwAddress: []byte(iface.HardwareAddr), 41 | SourceProtAddress: []byte(addr.IP.To4()), 42 | DstHwAddress: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, 43 | DstProtAddress: []byte(addr.IP.To4()), 44 | } 45 | // Set up buffer and options for serialization. 46 | buf := gopacket.NewSerializeBuffer() 47 | opts := gopacket.SerializeOptions{ 48 | FixLengths: true, 49 | ComputeChecksums: false, 50 | } 51 | // Send one packet for every address. 52 | gopacket.SerializeLayers(buf, opts, ð, &arp) 53 | if err := handle.WritePacketData(buf.Bytes()); err != nil { 54 | return err 55 | } 56 | return nil 57 | } 58 | 59 | func ArpAnnouncement(ifname string, addr *net.IPNet) error { 60 | iface, err := net.InterfaceByName(ifname) 61 | if err != nil { 62 | return err 63 | } 64 | handle, err := pcap.OpenLive(iface.Name, 65536, true, pcap.BlockForever) 65 | if err != nil { 66 | return err 67 | } 68 | defer handle.Close() 69 | return writeARP(handle, iface, addr) 70 | } 71 | 72 | // EnsureIPAssigned will check if ip is already present on a given link 73 | func EnsureIPAssigned(iface, cidr string) error { 74 | link, err := netlink.LinkByName(iface) 75 | if err != nil { 76 | return err 77 | } 78 | addr, err := netlink.ParseAddr(cidr) 79 | if err != nil { 80 | return err 81 | } 82 | addrList, err := netlink.AddrList(link, netlink.FAMILY_ALL) 83 | if err != nil { 84 | return err 85 | } 86 | for i := range addrList { 87 | if addrList[i].IPNet.String() == addr.IPNet.String() { 88 | return nil 89 | } 90 | } 91 | err = netlink.AddrAdd(link, addr) 92 | if err != nil { 93 | return err 94 | } 95 | if iface != "lo" { 96 | return ArpAnnouncement(iface, addr.IPNet) 97 | } 98 | return nil 99 | } 100 | 101 | // EnsureIPUnassigned ensure that given IP is not present on a given link 102 | func EnsureIPUnassigned(iface, cidr string) error { 103 | link, err := netlink.LinkByName(iface) 104 | if err != nil { 105 | return err 106 | } 107 | addr, err := netlink.ParseAddr(cidr) 108 | if err != nil { 109 | return err 110 | } 111 | addrList, err := netlink.AddrList(link, netlink.FAMILY_ALL) 112 | if err != nil { 113 | return err 114 | } 115 | for i := range addrList { 116 | if addrList[i].IPNet.String() == addr.IPNet.String() { 117 | return netlink.AddrDel(link, addr) 118 | } 119 | } 120 | return nil 121 | } 122 | 123 | type IPHandler interface { 124 | Add(iface, cidr string) error 125 | Del(iface, cidr string) error 126 | } 127 | 128 | type LinuxIPHandler struct{} 129 | 130 | func (l LinuxIPHandler) Add(iface, cidr string) error { 131 | glog.V(2).Infof("Adding addr %v on link %v", cidr, iface) 132 | return EnsureIPAssigned(iface, cidr) 133 | } 134 | func (l LinuxIPHandler) Del(iface, cidr string) error { 135 | glog.V(2).Infof("Removing addr %v from link %v", cidr, iface) 136 | return EnsureIPUnassigned(iface, cidr) 137 | } 138 | 139 | type AddCIDR struct { 140 | Cidr string 141 | } 142 | 143 | type DelCIDR struct { 144 | Cidr string 145 | } 146 | 147 | func IPIncrement(ip net.IP) { 148 | for j := len(ip) - 1; j >= 0; j-- { 149 | ip[j]++ 150 | if ip[j] > 0 { 151 | break 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /pkg/extensions/testing/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package testing 16 | 17 | import ( 18 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 19 | 20 | "github.com/stretchr/testify/mock" 21 | "k8s.io/client-go/1.5/pkg/api" 22 | "k8s.io/client-go/1.5/pkg/watch" 23 | ) 24 | 25 | type FakeExtClientset struct { 26 | mock.Mock 27 | Ipclaims *fakeIpClaims 28 | Ipnodes *fakeIpNodes 29 | Ipclaimpools *fakeIpClaimPools 30 | } 31 | 32 | func (f *FakeExtClientset) IPClaims() extensions.IPClaimsInterface { 33 | return f.Ipclaims 34 | } 35 | 36 | func (f *FakeExtClientset) IPNodes() extensions.IPNodesInterface { 37 | return f.Ipnodes 38 | } 39 | 40 | func (f *FakeExtClientset) IPClaimPools() extensions.IPClaimPoolsInterface { 41 | return f.Ipclaimpools 42 | } 43 | 44 | type fakeIpClaims struct { 45 | mock.Mock 46 | } 47 | 48 | func (f *fakeIpClaims) Get(name string) (*extensions.IpClaim, error) { 49 | args := f.Called(name) 50 | return args.Get(0).(*extensions.IpClaim), args.Error(1) 51 | } 52 | 53 | func (f *fakeIpClaims) Create(ipclaim *extensions.IpClaim) (*extensions.IpClaim, error) { 54 | args := f.Called(ipclaim) 55 | return ipclaim, args.Error(0) 56 | } 57 | 58 | func (f *fakeIpClaims) List(opts api.ListOptions) (*extensions.IpClaimList, error) { 59 | args := f.Called(opts) 60 | return args.Get(0).(*extensions.IpClaimList), args.Error(1) 61 | } 62 | 63 | func (f *fakeIpClaims) Update(ipclaim *extensions.IpClaim) (*extensions.IpClaim, error) { 64 | args := f.Called(ipclaim) 65 | return ipclaim, args.Error(0) 66 | } 67 | 68 | func (f *fakeIpClaims) Delete(name string, opts *api.DeleteOptions) error { 69 | args := f.Called(name, opts) 70 | return args.Error(0) 71 | } 72 | 73 | func (f *fakeIpClaims) Watch(_ api.ListOptions) (watch.Interface, error) { 74 | return nil, nil 75 | } 76 | 77 | type fakeIpNodes struct { 78 | mock.Mock 79 | } 80 | 81 | func (f *fakeIpNodes) Create(ipnode *extensions.IpNode) (*extensions.IpNode, error) { 82 | args := f.Called(ipnode) 83 | return ipnode, args.Error(0) 84 | } 85 | 86 | func (f *fakeIpNodes) List(opts api.ListOptions) (*extensions.IpNodeList, error) { 87 | args := f.Called(opts) 88 | return args.Get(0).(*extensions.IpNodeList), args.Error(1) 89 | } 90 | 91 | func (f *fakeIpNodes) Update(ipnode *extensions.IpNode) (*extensions.IpNode, error) { 92 | args := f.Called(ipnode) 93 | return args.Get(0).(*extensions.IpNode), args.Error(1) 94 | } 95 | 96 | func (f *fakeIpNodes) Delete(name string, opts *api.DeleteOptions) error { 97 | args := f.Called(name, opts) 98 | return args.Error(0) 99 | } 100 | 101 | func (f *fakeIpNodes) Get(name string) (*extensions.IpNode, error) { 102 | args := f.Called(name) 103 | return args.Get(0).(*extensions.IpNode), args.Error(1) 104 | } 105 | 106 | func (f *fakeIpNodes) Watch(_ api.ListOptions) (watch.Interface, error) { 107 | return nil, nil 108 | } 109 | 110 | type fakeIpClaimPools struct { 111 | mock.Mock 112 | } 113 | 114 | func (f *fakeIpClaimPools) Create(ipclaimpool *extensions.IpClaimPool) (*extensions.IpClaimPool, error) { 115 | args := f.Called(ipclaimpool) 116 | return ipclaimpool, args.Error(0) 117 | } 118 | 119 | func (f *fakeIpClaimPools) List(opts api.ListOptions) (*extensions.IpClaimPoolList, error) { 120 | args := f.Called(opts) 121 | return args.Get(0).(*extensions.IpClaimPoolList), args.Error(1) 122 | } 123 | 124 | func (f *fakeIpClaimPools) Update(ipclaimpool *extensions.IpClaimPool) (*extensions.IpClaimPool, error) { 125 | args := f.Called(ipclaimpool) 126 | return args.Get(0).(*extensions.IpClaimPool), args.Error(1) 127 | } 128 | 129 | func (f *fakeIpClaimPools) Delete(name string, opts *api.DeleteOptions) error { 130 | args := f.Called(name, opts) 131 | return args.Error(0) 132 | } 133 | 134 | func (f *fakeIpClaimPools) Get(name string) (*extensions.IpClaimPool, error) { 135 | args := f.Called(name) 136 | return args.Get(0).(*extensions.IpClaimPool), args.Error(1) 137 | } 138 | 139 | func NewFakeExtClientset() *FakeExtClientset { 140 | return &FakeExtClientset{ 141 | Ipclaims: &fakeIpClaims{}, 142 | Ipnodes: &fakeIpNodes{}, 143 | Ipclaimpools: &fakeIpClaimPools{}, 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /test/e2e/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "bytes" 19 | "flag" 20 | "fmt" 21 | "io" 22 | "net/url" 23 | "strings" 24 | "time" 25 | 26 | "k8s.io/client-go/1.5/kubernetes" 27 | "k8s.io/client-go/1.5/pkg/api" 28 | "k8s.io/client-go/1.5/pkg/api/v1" 29 | "k8s.io/client-go/1.5/rest" 30 | "k8s.io/client-go/1.5/tools/clientcmd" 31 | "k8s.io/kubernetes/pkg/client/unversioned/remotecommand" 32 | remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" 33 | "k8s.io/kubernetes/pkg/util/httpstream/spdy" 34 | 35 | . "github.com/onsi/ginkgo" 36 | . "github.com/onsi/gomega" 37 | ) 38 | 39 | var MASTER string 40 | var TESTLINK string 41 | 42 | func init() { 43 | flag.StringVar(&MASTER, "master", "http://apiserver:8888", "apiserver address to use with restclient") 44 | flag.StringVar(&TESTLINK, "testlink", "eth0", "link to use on the side of tests") 45 | } 46 | 47 | func GetTestLink() string { 48 | return TESTLINK 49 | } 50 | 51 | func Logf(format string, a ...interface{}) { 52 | fmt.Fprintf(GinkgoWriter, format, a...) 53 | } 54 | 55 | func LoadConfig() *rest.Config { 56 | config, err := clientcmd.BuildConfigFromFlags(MASTER, "") 57 | Expect(err).NotTo(HaveOccurred()) 58 | return config 59 | } 60 | 61 | func KubeClient() (*kubernetes.Clientset, error) { 62 | Logf("Using master %v\n", MASTER) 63 | config := LoadConfig() 64 | clientset, err := kubernetes.NewForConfig(config) 65 | Expect(err).NotTo(HaveOccurred()) 66 | return clientset, nil 67 | } 68 | 69 | func WaitForReady(clientset *kubernetes.Clientset, pod *v1.Pod) { 70 | Eventually(func() error { 71 | podUpdated, err := clientset.Core().Pods(pod.Namespace).Get(pod.Name) 72 | if err != nil { 73 | return err 74 | } 75 | if podUpdated.Status.Phase != v1.PodRunning { 76 | return fmt.Errorf("pod %v is not running phase: %v", podUpdated.Name, podUpdated.Status.Phase) 77 | } 78 | return nil 79 | }, 120*time.Second, 5*time.Second).Should(BeNil()) 80 | } 81 | 82 | func DumpLogs(clientset *kubernetes.Clientset, pods ...v1.Pod) { 83 | for _, pod := range pods { 84 | dumpLogs(clientset, pod) 85 | } 86 | } 87 | 88 | func dumpLogs(clientset *kubernetes.Clientset, pod v1.Pod) { 89 | req := clientset.Core().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}) 90 | readCloser, err := req.Stream() 91 | Expect(err).NotTo(HaveOccurred()) 92 | defer readCloser.Close() 93 | Logf("\n Dumping logs for %v:%v \n", pod.Namespace, pod.Name) 94 | _, err = io.Copy(GinkgoWriter, readCloser) 95 | Expect(err).NotTo(HaveOccurred()) 96 | } 97 | 98 | func ExecInPod(clientset *kubernetes.Clientset, pod v1.Pod, cmd ...string) (string, string, error) { 99 | Logf("Running %v in %v\n", cmd, pod.Name) 100 | 101 | container := pod.Spec.Containers[0].Name 102 | var stdout, stderr bytes.Buffer 103 | config := LoadConfig() 104 | rest := clientset.CoreClient.GetRESTClient() 105 | req := rest.Post(). 106 | Resource("pods"). 107 | Name(pod.Name). 108 | Namespace(pod.Namespace). 109 | SubResource("exec"). 110 | Param("container", container) 111 | req.VersionedParams(&api.PodExecOptions{ 112 | Container: container, 113 | Command: cmd, 114 | TTY: false, 115 | Stdin: false, 116 | Stdout: true, 117 | Stderr: true, 118 | }, api.ParameterCodec) 119 | err := execute("POST", req.URL(), config, nil, &stdout, &stderr, false) 120 | Logf("Error %v: %v\n", cmd, stderr.String()) 121 | Logf("Output %v: %v\n", cmd, stdout.String()) 122 | return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err 123 | } 124 | 125 | func execute(method string, url *url.URL, config *rest.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { 126 | tlsConfig, err := rest.TLSConfigFor(config) 127 | if err != nil { 128 | return err 129 | } 130 | upgrader := spdy.NewRoundTripper(tlsConfig) 131 | exec, err := remotecommand.NewStreamExecutor(upgrader, nil, method, url) 132 | if err != nil { 133 | return err 134 | } 135 | return exec.Stream(remotecommand.StreamOptions{ 136 | SupportedProtocols: remotecommandserver.SupportedStreamingProtocols, 137 | Stdin: stdin, 138 | Stdout: stdout, 139 | Stderr: stderr, 140 | Tty: tty, 141 | }) 142 | } 143 | -------------------------------------------------------------------------------- /doc/operating-modes.md: -------------------------------------------------------------------------------- 1 | Basic Modules and Operating Modes 2 | ================================= 3 | 4 | ## Basic Modules 5 | 6 | Application consists of two basic modules: controller and scheduler. 7 | 8 | Controller module manages IPs on its node (brings up, deletes, ensures that list 9 | of IPs on node reflects the list of IPs required for services). 10 | Controller(s) should be run on nodes that have external connectivity as 11 | External IPs will be spawned on those nodes. Each node should have no more than 12 | one controller at a time. 13 | 14 | Scheduler module processes IP claims from services and distributes them among 15 | the controllers (i.e. nodes). 16 | Scheduler(s) can be run on any nodes as they just schedule claims among the 17 | controllers. Scheduler module is not obligatory to be run. It is not in use 18 | while application runs in Simple mode. 19 | 20 | ## Operating Modes 21 | 22 | External IP Controller application may be run in one of the operating modes: 23 | * [Simple](#simple-mode) 24 | * [Claims](#claims-mode) 25 | 26 | ## Simple Mode 27 | 28 | External IP controller application (its controller module) will be run on one of 29 | the nodes. It should be run on node that has external connectivity as all the 30 | External IPs will be spawned on that node. 31 | Kubernetes provides fail-over for External IP controller application. So, when 32 | there's a problem with k8s node, External IP controller will be spawned on 33 | another k8s worker node and will bring External IPs up on that node. 34 | Simple mode is easy to setup and takes less resources. It makes sense when all 35 | IPs should be brought up on the same node. However, fail-over in this mode takes 36 | longer than in Claims mode (k8s detects node failure in much longer intervals by 37 | default, this could be optimized - see "Fail-over optimization on k8s side" document) 38 | and it may work wrong in some cases. 39 | 40 | # Parameters 41 | 42 | Next command-line parameters are available in Simple mode for controller module: 43 | * `iface` - interface that will be used to assign IP addresses (default "eth0"). 44 | * `kubeconfig` - kubeconfig to use with kubernetes client (default ""; incluster 45 | configuration for auth will be used by default). 46 | * `mask` - mask part of network CIDR (default "32"). 47 | * `resync` - interval to resync state for all ips (default 20 sec). 48 | It is usually enough to set `iface` and `mask` parameters. 49 | 50 | ## Claims Mode 51 | 52 | External IP controller application will be run on several nodes. One or more 53 | controller modules and one or more scheduler modules will be run. Controller 54 | modules should be run on nodes where IPs are expected to be spawned. Scheduler 55 | modules can be run on any nodes. There is no much sense to run more than one 56 | scheduler on every particular node. Several scheduler modules are run to provide 57 | HA for scheduler (A/B mode). So, that in case of no response from active 58 | scheduler (e.g. on node failure) another one becomes active. If more than one 59 | scheduler will be used then scheduler election mode should be switched on 60 | (parameter `leader-elect=true`) otherwise there can be race conditions between 61 | schedulers. 62 | It is better not to run scheduler on same node as controller because in case of 63 | node outage IPs rescheduling will take more time. 64 | 65 | # IPs Distribution in Claims Mode 66 | 67 | There can be different rules of IPs distribution among controllers (i.e. nodes) 68 | in Claims mode. This is controlled by `nodefilter` parameter. Default rule 69 | is to distribute IPs evenly among all the controllers (`nodefilter=fair`). 70 | Alternative rule is `nodefilter=first-alive` where all IPs will be spawned on 71 | the first available controller (i.e. node). Claims mode with `first-alive` 72 | rule is similar to Simple mode but with more responsive and correct fail-over. 73 | 74 | # Parameters 75 | 76 | Next command-line parameters are available in Claims mode for controller module: 77 | * `iface` - interface that will be used to assign IP addresses (default "eth0"). 78 | * `hb` - how often to send heartbeats from controllers (default 2 sec). 79 | * `kubeconfig` - kubeconfig to use with kubernetes client (default ""; incluster 80 | configuration for auth will be used by default). 81 | * `resync` - interval to resync state for all IPs (default 20 sec). 82 | * `hostname` - use provided hostname instead of os.Hostname (default 83 | os.Hostname). 84 | 85 | Next command-line parameters are available in Claims mode for scheduler module: 86 | * `kubeconfig` - kubeconfig to use with kubernetes client (default ""; incluster 87 | configuration for auth will be used by default). 88 | * `mask` - mask part of network CIDR (default "32"), it is not in use for 89 | auto-allocation. 90 | * `nodefilter` - node filter to use while dispatching IP claims; it controls IPs 91 | distribution between controllers (default "fair"). 92 | * `monitor` - how often to check controllers responsiveness (default 4 93 | sec). 94 | 95 | -------------------------------------------------------------------------------- /test/integration/network_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | "strings" 21 | "time" 22 | 23 | controller "github.com/Mirantis/k8s-externalipcontroller/pkg" 24 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 25 | 26 | "k8s.io/client-go/1.5/pkg/api/v1" 27 | fcache "k8s.io/client-go/1.5/tools/cache/testing" 28 | 29 | "github.com/vishvananda/netlink" 30 | 31 | "reflect" 32 | 33 | . "github.com/onsi/ginkgo" 34 | . "github.com/onsi/gomega" 35 | ) 36 | 37 | var _ = Describe("Network [sudo]", func() { 38 | 39 | var linkNames []string 40 | 41 | BeforeEach(func() { 42 | linkNames = []string{"test11", "test12"} 43 | ensureLinksRemoved(linkNames...) 44 | }) 45 | 46 | AfterEach(func() { 47 | ensureLinksRemoved(linkNames...) 48 | }) 49 | 50 | It("Multiple ips can be assigned", func() { 51 | link := &netlink.Dummy{netlink.LinkAttrs{Name: linkNames[0]}} 52 | By("adding dummy link with name " + link.Attrs().Name) 53 | Expect(netlink.LinkAdd(link)).NotTo(HaveOccurred()) 54 | By("getting link up") 55 | Expect(netlink.LinkSetUp(link)).NotTo(HaveOccurred()) 56 | cidrToAssign := []string{"10.10.0.2/24", "10.10.0.2/24", "10.10.0.3/24"} 57 | for _, cidr := range cidrToAssign { 58 | err := netutils.EnsureIPAssigned(link.Attrs().Name, cidr) 59 | Expect(err).NotTo(HaveOccurred()) 60 | } 61 | addrList, err := netlink.AddrList(link, netlink.FAMILY_V4) 62 | Expect(err).NotTo(HaveOccurred()) 63 | ipSet := map[string]bool{} 64 | expectedIpSet := map[string]bool{"10.10.0.2/24": true, "10.10.0.3/24": true} 65 | for i := range addrList { 66 | ipSet[addrList[i].IPNet.String()] = true 67 | } 68 | Expect(expectedIpSet).To(BeEquivalentTo(ipSet)) 69 | }) 70 | 71 | It("Controller with noop manager will create provided externalIPs", func() { 72 | link := &netlink.Dummy{netlink.LinkAttrs{Name: linkNames[0]}} 73 | 74 | By("adding link for controller") 75 | Expect(netlink.LinkAdd(link)).NotTo(HaveOccurred()) 76 | Expect(netlink.LinkSetUp(link)).NotTo(HaveOccurred()) 77 | 78 | By("creating and running controller with fake source") 79 | stop := make(chan struct{}) 80 | defer close(stop) 81 | source := fcache.NewFakeControllerSource() 82 | c := controller.NewExternalIpControllerWithSource("1", link.Attrs().Name, "24", source) 83 | go c.Run(stop) 84 | 85 | testIps := [][]string{ 86 | {"10.10.0.2", "10.10.0.3"}, 87 | {"10.10.0.2", "10.10.0.3", "10.10.0.4"}, 88 | {"10.10.0.5"}, 89 | } 90 | services := map[string]*v1.Service{} 91 | expectedIps := map[string]bool{} 92 | for i, ips := range testIps { 93 | for _, ip := range ips { 94 | expectedIps[strings.Join([]string{ip, c.Mask}, "/")] = true 95 | } 96 | 97 | svc := &v1.Service{ 98 | ObjectMeta: v1.ObjectMeta{Name: "service-" + strconv.Itoa(i)}, 99 | Spec: v1.ServiceSpec{ExternalIPs: ips}, 100 | } 101 | services[svc.Name] = svc 102 | source.Add(svc) 103 | } 104 | By("waiting until ips will be assigned") 105 | verifyAddrs(link, expectedIps) 106 | By("updating service with new ip list and waiting until ips assignment will be updated") 107 | svcToUpdate := services["service-2"] 108 | svcToUpdate.Spec.ExternalIPs = []string{"10.10.0.7"} 109 | source.Modify(svcToUpdate) 110 | expectedIps["10.10.0.7/24"] = true 111 | delete(expectedIps, "10.10.0.5/24") 112 | verifyAddrs(link, expectedIps) 113 | By("removing service with single ip and waiting until this ip won't be on a link") 114 | source.Delete(services["service-2"]) 115 | delete(expectedIps, "10.10.0.7/24") 116 | verifyAddrs(link, expectedIps) 117 | By("removing service with ips that are assigned to some other service and confirming that they are still on link") 118 | source.Delete(services["service-1"]) 119 | delete(expectedIps, "10.10.0.4/24") 120 | verifyAddrs(link, expectedIps) 121 | }) 122 | }) 123 | 124 | func verifyAddrs(link netlink.Link, expectedIps map[string]bool) { 125 | Eventually(func() error { 126 | resultIps := map[string]bool{} 127 | addrList, err := netlink.AddrList(link, netlink.FAMILY_V4) 128 | if err != nil { 129 | return err 130 | } 131 | for _, addr := range addrList { 132 | resultIps[addr.IPNet.String()] = true 133 | } 134 | if !reflect.DeepEqual(expectedIps, resultIps) { 135 | return fmt.Errorf("Assigned ips %v are not equal to expected %v.", resultIps, expectedIps) 136 | } 137 | return nil 138 | }, 20*time.Second, 1*time.Second).Should(BeNil()) 139 | } 140 | 141 | func ensureLinksRemoved(links ...string) { 142 | for _, l := range links { 143 | link, err := netlink.LinkByName(l) 144 | if err != nil { 145 | continue 146 | } 147 | netlink.LinkDel(link) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /pkg/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package externalip 16 | 17 | import ( 18 | "reflect" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 23 | "github.com/golang/glog" 24 | 25 | "k8s.io/client-go/1.5/kubernetes" 26 | "k8s.io/client-go/1.5/pkg/api" 27 | "k8s.io/client-go/1.5/pkg/api/v1" 28 | "k8s.io/client-go/1.5/pkg/runtime" 29 | "k8s.io/client-go/1.5/pkg/watch" 30 | "k8s.io/client-go/1.5/rest" 31 | "k8s.io/client-go/1.5/tools/cache" 32 | ) 33 | 34 | type ExternalIpController struct { 35 | Uid string 36 | Iface string 37 | Mask string 38 | 39 | source cache.ListerWatcher 40 | ipHandler netutils.IPHandler 41 | Queue workqueue.QueueType 42 | 43 | resyncInterval time.Duration 44 | } 45 | 46 | func NewExternalIpController(config *rest.Config, uid, iface, mask string, resyncInterval time.Duration) (*ExternalIpController, error) { 47 | clientset, err := kubernetes.NewForConfig(config) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | lw := &cache.ListWatch{ 53 | ListFunc: func(options api.ListOptions) (runtime.Object, error) { 54 | return clientset.Core().Services(api.NamespaceAll).List(api.ListOptions{}) 55 | }, 56 | WatchFunc: func(options api.ListOptions) (watch.Interface, error) { 57 | return clientset.Core().Services(api.NamespaceAll).Watch(api.ListOptions{}) 58 | }, 59 | } 60 | return &ExternalIpController{ 61 | Uid: uid, 62 | Iface: iface, 63 | Mask: mask, 64 | source: lw, 65 | ipHandler: netutils.LinuxIPHandler{}, 66 | Queue: workqueue.NewQueue(), 67 | resyncInterval: resyncInterval, 68 | }, nil 69 | } 70 | 71 | func NewExternalIpControllerWithSource(uid, iface, mask string, source cache.ListerWatcher) *ExternalIpController { 72 | return &ExternalIpController{ 73 | Uid: uid, 74 | Iface: iface, 75 | Mask: mask, 76 | source: source, 77 | ipHandler: netutils.LinuxIPHandler{}, 78 | Queue: workqueue.NewQueue(), 79 | } 80 | } 81 | 82 | func (c *ExternalIpController) Run(stopCh chan struct{}) { 83 | glog.Infof("Starting externalipcontroller") 84 | var store cache.Store 85 | store, controller := cache.NewInformer( 86 | c.source, 87 | &v1.Service{}, 88 | c.resyncInterval, 89 | cache.ResourceEventHandlerFuncs{ 90 | AddFunc: func(obj interface{}) { 91 | c.processServiceExternalIPs(nil, obj.(*v1.Service), store) 92 | }, 93 | UpdateFunc: func(old, cur interface{}) { 94 | c.processServiceExternalIPs(old.(*v1.Service), cur.(*v1.Service), store) 95 | }, 96 | DeleteFunc: func(obj interface{}) { 97 | c.processServiceExternalIPs(obj.(*v1.Service), nil, store) 98 | }, 99 | }, 100 | ) 101 | 102 | // we can spawn worker for each interface, but i doubt that we will ever need such 103 | // optimization 104 | go c.worker() 105 | go controller.Run(stopCh) 106 | <-stopCh 107 | c.Queue.Close() 108 | } 109 | 110 | func (c *ExternalIpController) worker() { 111 | for { 112 | item, quit := c.Queue.Get() 113 | if quit { 114 | return 115 | } 116 | c.processItem(item) 117 | } 118 | } 119 | 120 | func (c *ExternalIpController) processItem(item interface{}) { 121 | defer c.Queue.Done(item) 122 | var err error 123 | var cidr string 124 | var action string 125 | switch t := item.(type) { 126 | case *netutils.AddCIDR: 127 | err = c.ipHandler.Add(c.Iface, t.Cidr) 128 | cidr = t.Cidr 129 | action = "assignment" 130 | 131 | case *netutils.DelCIDR: 132 | err = c.ipHandler.Del(c.Iface, t.Cidr) 133 | cidr = t.Cidr 134 | action = "removal" 135 | } 136 | if err != nil { 137 | glog.Errorf("Error during %s of IP %v on %s - %v", action, cidr, c.Iface, err) 138 | c.Queue.Add(item) 139 | } else { 140 | glog.V(2).Infof("%s of IP %v was done successfully", action, cidr) 141 | } 142 | } 143 | 144 | func boolMapDifference(minuend, subtrahend map[string]bool) map[string]bool { 145 | difference := make(map[string]bool) 146 | 147 | for key := range minuend { 148 | if !subtrahend[key] { 149 | difference[key] = true 150 | } 151 | } 152 | 153 | return difference 154 | } 155 | 156 | func neglectIPsInUse(ips map[string]bool, key string, store cache.Store) { 157 | svcList := store.List() 158 | for s := range svcList { 159 | svc := svcList[s].(*v1.Service) 160 | svcKey, _ := cache.MetaNamespaceKeyFunc(svc) 161 | if svcKey != key { 162 | for _, ip := range svc.Spec.ExternalIPs { 163 | delete(ips, ip) 164 | } 165 | } 166 | } 167 | } 168 | 169 | func (c *ExternalIpController) processServiceExternalIPs(old, cur *v1.Service, store cache.Store) { 170 | old_ips := make(map[string]bool) 171 | cur_ips := make(map[string]bool) 172 | key := "" 173 | 174 | if old != nil { 175 | for i := range old.Spec.ExternalIPs { 176 | old_ips[old.Spec.ExternalIPs[i]] = true 177 | } 178 | key, _ = cache.MetaNamespaceKeyFunc(old) 179 | } 180 | if cur != nil { 181 | for i := range cur.Spec.ExternalIPs { 182 | cur_ips[cur.Spec.ExternalIPs[i]] = true 183 | } 184 | key, _ = cache.MetaNamespaceKeyFunc(cur) 185 | } 186 | 187 | if reflect.DeepEqual(cur_ips, old_ips) { 188 | return 189 | } 190 | 191 | ips_to_add := boolMapDifference(cur_ips, old_ips) 192 | ips_to_remove := boolMapDifference(old_ips, cur_ips) 193 | 194 | neglectIPsInUse(ips_to_add, key, store) 195 | neglectIPsInUse(ips_to_remove, key, store) 196 | 197 | for ip := range ips_to_add { 198 | cidr := ip + "/" + c.Mask 199 | c.Queue.Add(&netutils.AddCIDR{cidr}) 200 | } 201 | for ip := range ips_to_remove { 202 | cidr := ip + "/" + c.Mask 203 | c.Queue.Add(&netutils.DelCIDR{cidr}) 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /pkg/claimcontroller/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package claimcontroller 16 | 17 | import ( 18 | "time" 19 | "strings" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 23 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 24 | 25 | "github.com/golang/glog" 26 | "k8s.io/client-go/1.5/kubernetes" 27 | "k8s.io/client-go/1.5/pkg/api" 28 | "k8s.io/client-go/1.5/pkg/api/errors" 29 | "k8s.io/client-go/1.5/pkg/runtime" 30 | "k8s.io/client-go/1.5/pkg/watch" 31 | "k8s.io/client-go/1.5/rest" 32 | "k8s.io/client-go/1.5/tools/cache" 33 | ) 34 | 35 | func NewClaimController(iface, uid string, config *rest.Config, resyncInterval time.Duration, hbInterval time.Duration) (*claimController, error) { 36 | clientset, err := kubernetes.NewForConfig(config) 37 | if err != nil { 38 | return nil, err 39 | } 40 | ext, err := extensions.WrapClientsetWithExtensions(clientset, config) 41 | if err != nil { 42 | return nil, err 43 | } 44 | claimSource := &cache.ListWatch{ 45 | ListFunc: func(options api.ListOptions) (runtime.Object, error) { 46 | return ext.IPClaims().List(options) 47 | }, 48 | WatchFunc: func(options api.ListOptions) (watch.Interface, error) { 49 | glog.V(3).Infof("Calling claim watcher with options %v", options) 50 | return ext.IPClaims().Watch(options) 51 | }, 52 | } 53 | queue := workqueue.NewQueue() 54 | uid = strings.Replace(uid, ".", "-", -1) 55 | return &claimController{ 56 | Clientset: clientset, 57 | ExtensionsClientset: ext, 58 | Iface: iface, 59 | Uid: uid, 60 | claimSource: claimSource, 61 | queue: queue, 62 | iphandler: netutils.LinuxIPHandler{}, 63 | heartbeatPeriod: hbInterval, 64 | resyncInterval: resyncInterval, 65 | }, nil 66 | } 67 | 68 | type claimController struct { 69 | Clientset *kubernetes.Clientset 70 | ExtensionsClientset extensions.ExtensionsClientset 71 | // i am not sure that it should be configurable for controller 72 | Iface string 73 | Uid string 74 | 75 | claimSource cache.ListerWatcher 76 | claimStore cache.Store 77 | 78 | queue workqueue.QueueType 79 | iphandler netutils.IPHandler 80 | 81 | // heartbeatPeriod for a node, should be < monitorPeriod in scheduller 82 | heartbeatPeriod time.Duration 83 | 84 | resyncInterval time.Duration 85 | } 86 | 87 | func (c *claimController) Run(stop chan struct{}) { 88 | go c.worker() 89 | go c.claimWatcher(stop) 90 | go c.heartbeatIpNode(stop, time.Tick(c.heartbeatPeriod)) 91 | <-stop 92 | c.queue.Close() 93 | } 94 | 95 | func (c *claimController) claimWatcher(stop chan struct{}) { 96 | store, controller := cache.NewInformer( 97 | c.claimSource, 98 | &extensions.IpClaim{}, 99 | c.resyncInterval, 100 | cache.ResourceEventHandlerFuncs{ 101 | AddFunc: func(obj interface{}) { 102 | claim := obj.(*extensions.IpClaim) 103 | glog.V(3).Infof("Received add event for ipclaim %v - %v - %v", 104 | claim.Metadata.Name, claim.Spec.NodeName, claim.Metadata.ResourceVersion) 105 | c.queue.Add(claim) 106 | }, 107 | UpdateFunc: func(old, cur interface{}) { 108 | oldClaim := old.(*extensions.IpClaim) 109 | curClaim := cur.(*extensions.IpClaim) 110 | glog.V(3).Infof("Received update event. Old ipclaim %v - %v, New ipclaim %v - %v -%v", 111 | oldClaim.Metadata.Name, oldClaim.Spec.NodeName, 112 | curClaim.Metadata.Name, curClaim.Spec.NodeName, curClaim.Metadata.ResourceVersion) 113 | c.queue.Add(curClaim) 114 | }, 115 | DeleteFunc: func(obj interface{}) { 116 | claim := obj.(*extensions.IpClaim) 117 | glog.V(3).Infof("Received delete event for %v - %v. Resource version %v", 118 | claim.Metadata.Name, claim.Spec.NodeName, claim.Metadata.ResourceVersion) 119 | c.queue.Add(claim) 120 | }, 121 | }, 122 | ) 123 | c.claimStore = store 124 | controller.Run(stop) 125 | } 126 | 127 | func (c *claimController) worker() { 128 | for { 129 | item, quit := c.queue.Get() 130 | if quit { 131 | return 132 | } 133 | err := c.processClaim(item.(*extensions.IpClaim)) 134 | if err != nil { 135 | glog.Errorf("Error processing claim %v", err) 136 | c.queue.Add(item) 137 | } 138 | c.queue.Done(item) 139 | } 140 | } 141 | 142 | func (c *claimController) processClaim(ipclaim *extensions.IpClaim) error { 143 | glog.V(5).Infof("Processing claim %v with node %v and uid %v", 144 | ipclaim.Spec.Cidr, ipclaim.Spec.NodeName, c.Uid) 145 | if _, exists, _ := c.claimStore.Get(ipclaim); !exists { 146 | return c.iphandler.Del(c.Iface, ipclaim.Spec.Cidr) 147 | } 148 | if ipclaim.Spec.NodeName == c.Uid { 149 | return c.iphandler.Add(c.Iface, ipclaim.Spec.Cidr) 150 | } else { 151 | return c.iphandler.Del(c.Iface, ipclaim.Spec.Cidr) 152 | } 153 | } 154 | 155 | func (c *claimController) heartbeatIpNode(stop chan struct{}, ticker <-chan time.Time) { 156 | for { 157 | select { 158 | case <-stop: 159 | return 160 | case <-ticker: 161 | ipnode, err := c.ExtensionsClientset.IPNodes().Get(c.Uid) 162 | if errors.IsNotFound(err) { 163 | ipnode := &extensions.IpNode{ 164 | Metadata: api.ObjectMeta{Name: c.Uid}, 165 | } 166 | _, err := c.ExtensionsClientset.IPNodes().Create(ipnode) 167 | if err != nil { 168 | glog.Errorf("Error creating node %v : %v", c.Uid, err) 169 | } 170 | continue 171 | } 172 | 173 | if err != nil { 174 | glog.Errorf("Error fetching node %v : %v", c.Uid, err) 175 | continue 176 | } 177 | glog.V(3).Infof("Updating ipnode %v. Version %v.", 178 | ipnode.Metadata.Name, ipnode.Revision) 179 | ipnode.Revision++ 180 | _, err = c.ExtensionsClientset.IPNodes().Update(ipnode) 181 | if err != nil { 182 | glog.Errorf("Error updating node %v : %v", c.Uid, err) 183 | continue 184 | } 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /pkg/extensions/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "encoding/json" 19 | "errors" 20 | "net" 21 | 22 | "k8s.io/client-go/1.5/pkg/api" 23 | "k8s.io/client-go/1.5/pkg/api/meta" 24 | "k8s.io/client-go/1.5/pkg/api/unversioned" 25 | "k8s.io/client-go/1.5/pkg/apimachinery/announced" 26 | "k8s.io/client-go/1.5/pkg/runtime" 27 | 28 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 29 | ) 30 | 31 | const ( 32 | GroupName string = "ipcontroller.ext" 33 | Version string = "v1" 34 | ) 35 | 36 | var ( 37 | SchemeGroupVersion = unversioned.GroupVersion{Group: GroupName, Version: Version} 38 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 39 | ) 40 | 41 | func addKnownTypes(scheme *runtime.Scheme) error { 42 | scheme.AddKnownTypes( 43 | SchemeGroupVersion, 44 | &IpNode{}, 45 | &IpNodeList{}, 46 | &IpClaim{}, 47 | &IpClaimList{}, 48 | &IpClaimPool{}, 49 | &IpClaimPoolList{}, 50 | 51 | &api.ListOptions{}, 52 | &api.DeleteOptions{}, 53 | ) 54 | return nil 55 | } 56 | 57 | func init() { 58 | if err := announced.NewGroupMetaFactory( 59 | &announced.GroupMetaFactoryArgs{ 60 | GroupName: GroupName, 61 | VersionPreferenceOrder: []string{SchemeGroupVersion.Version}, 62 | AddInternalObjectsToScheme: SchemeBuilder.AddToScheme, 63 | }, 64 | announced.VersionToSchemeFunc{ 65 | SchemeGroupVersion.Version: SchemeBuilder.AddToScheme, 66 | }, 67 | ).Announce().RegisterAndEnable(); err != nil { 68 | panic(err) 69 | } 70 | } 71 | 72 | type IpNode struct { 73 | unversioned.TypeMeta `json:",inline"` 74 | 75 | // Standard object metadata 76 | Metadata api.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 77 | 78 | // used as a heartbeat 79 | Revision int64 `json:"generation,omitempty"` 80 | } 81 | 82 | func (e *IpNode) GetObjectKind() unversioned.ObjectKind { 83 | return &e.TypeMeta 84 | } 85 | 86 | func (e *IpNode) GetObjectMeta() meta.Object { 87 | return &e.Metadata 88 | } 89 | 90 | type IpNodeList struct { 91 | unversioned.TypeMeta `json:",inline"` 92 | 93 | // Standard list metadata. 94 | unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 95 | 96 | Items []IpNode `json:"items" protobuf:"bytes,2,rep,name=items"` 97 | } 98 | 99 | type IpClaim struct { 100 | unversioned.TypeMeta `json:",inline"` 101 | 102 | // Standard object metadata 103 | Metadata api.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 104 | 105 | Spec IpClaimSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` 106 | } 107 | 108 | func (e *IpClaim) GetObjectKind() unversioned.ObjectKind { 109 | return &e.TypeMeta 110 | } 111 | 112 | func (e *IpClaim) GetObjectMeta() meta.Object { 113 | return &e.Metadata 114 | } 115 | 116 | type IpClaimList struct { 117 | unversioned.TypeMeta `json:",inline"` 118 | 119 | // Standard list metadata. 120 | unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 121 | 122 | Items []IpClaim `json:"items" protobuf:"bytes,2,rep,name=items"` 123 | } 124 | 125 | type IpClaimSpec struct { 126 | // NodeName used to identify where IPClaim is assigned (IPNode.Name) 127 | NodeName string `json:"nodeName" protobuf:"bytes,10,opt,name=nodeName"` 128 | Cidr string `json:"cidr,omitempty" protobuf:"bytes,10,opt,name=cidr"` 129 | Link string `json:"link" protobuf:"bytes,10,opt,name=link"` 130 | } 131 | 132 | type IpClaimPool struct { 133 | unversioned.TypeMeta `json:",inline"` 134 | 135 | // Standard object metadata 136 | Metadata api.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 137 | 138 | Spec IpClaimPoolSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` 139 | } 140 | 141 | type IpClaimPoolSpec struct { 142 | CIDR string `json:"cidr" protobuf:"bytes,10,opt,name=cidr"` 143 | Ranges [][]string `json:"ranges,omitempty" protobuf:"bytes,5,opt,name=ranges"` 144 | Allocated map[string]string `json:"allocated,omitempty" protobuf:"bytes,2,opt,name=allocated"` 145 | } 146 | 147 | type IpClaimPoolList struct { 148 | unversioned.TypeMeta `json:",inline"` 149 | 150 | // Standard list metadata. 151 | unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 152 | 153 | Items []IpClaimPool `json:"items" protobuf:"bytes,2,rep,name=items"` 154 | } 155 | 156 | func (p *IpClaimPool) AvailableIP() (availableIP string, err error) { 157 | ip, network, err := net.ParseCIDR(p.Spec.CIDR) 158 | if err != nil { 159 | return 160 | } 161 | 162 | var dropOffIP net.IP 163 | 164 | ranges := [][]net.IP{} 165 | 166 | //in case 'Ranges' is not set for the pool assume ranges of the 167 | //network itself 168 | if p.Spec.Ranges != nil { 169 | for _, r := range p.Spec.Ranges { 170 | ip = net.ParseIP(r[0]) 171 | dropOffIP = net.ParseIP(r[len(r)-1]) 172 | netutils.IPIncrement(dropOffIP) 173 | 174 | ranges = append(ranges, []net.IP{ip, dropOffIP}) 175 | } 176 | } else { 177 | //network address is not usable 178 | netutils.IPIncrement(ip) 179 | ranges = [][]net.IP{{ip, dropOffIP}} 180 | } 181 | 182 | for _, r := range ranges { 183 | curAddr := r[0] 184 | nextAddr := make(net.IP, len(curAddr)) 185 | copy(nextAddr, curAddr) 186 | netutils.IPIncrement(nextAddr) 187 | 188 | firstOut := r[len(r)-1] 189 | 190 | for network.Contains(curAddr) && network.Contains(nextAddr) && !curAddr.Equal(firstOut) { 191 | if _, exists := p.Spec.Allocated[curAddr.String()]; !exists { 192 | return curAddr.String(), nil 193 | } 194 | netutils.IPIncrement(curAddr) 195 | netutils.IPIncrement(nextAddr) 196 | 197 | } 198 | } 199 | 200 | return "", errors.New("There is no free IP left in the pool") 201 | } 202 | 203 | func (p *IpClaimPool) GetObjectKind() unversioned.ObjectKind { 204 | return &p.TypeMeta 205 | } 206 | 207 | func (p *IpClaimPool) GetObjectMeta() meta.Object { 208 | return &p.Metadata 209 | } 210 | 211 | // see https://github.com/kubernetes/client-go/issues/8 212 | type ExampleIpNode IpNode 213 | type ExampleIpNodesList IpNodeList 214 | type ExampleIpClaim IpClaim 215 | type ExampleIpClaimList IpClaimList 216 | type ExampleIpClaimPool IpClaimPool 217 | type ExampleIpClaimPoolList IpClaimPoolList 218 | 219 | func (e *IpClaimPool) UnmarshalJSON(data []byte) error { 220 | tmp := ExampleIpClaimPool{} 221 | err := json.Unmarshal(data, &tmp) 222 | if err != nil { 223 | return err 224 | } 225 | tmp2 := IpClaimPool(tmp) 226 | *e = tmp2 227 | return nil 228 | } 229 | 230 | func (e *IpClaimPoolList) UnmarshalJSON(data []byte) error { 231 | tmp := ExampleIpClaimPoolList{} 232 | err := json.Unmarshal(data, &tmp) 233 | if err != nil { 234 | return err 235 | } 236 | tmp2 := IpClaimPoolList(tmp) 237 | *e = tmp2 238 | return nil 239 | } 240 | 241 | func (e *IpNode) UnmarshalJSON(data []byte) error { 242 | tmp := ExampleIpNode{} 243 | err := json.Unmarshal(data, &tmp) 244 | if err != nil { 245 | return err 246 | } 247 | tmp2 := IpNode(tmp) 248 | *e = tmp2 249 | return nil 250 | } 251 | 252 | func (el *IpNodeList) UnmarshalJSON(data []byte) error { 253 | tmp := ExampleIpNodesList{} 254 | err := json.Unmarshal(data, &tmp) 255 | if err != nil { 256 | return err 257 | } 258 | tmp2 := IpNodeList(tmp) 259 | *el = tmp2 260 | return nil 261 | } 262 | 263 | func (e *IpClaim) UnmarshalJSON(data []byte) error { 264 | tmp := ExampleIpClaim{} 265 | err := json.Unmarshal(data, &tmp) 266 | if err != nil { 267 | return err 268 | } 269 | tmp2 := IpClaim(tmp) 270 | *e = tmp2 271 | return nil 272 | } 273 | 274 | func (el *IpClaimList) UnmarshalJSON(data []byte) error { 275 | tmp := ExampleIpClaimList{} 276 | err := json.Unmarshal(data, &tmp) 277 | if err != nil { 278 | return err 279 | } 280 | tmp2 := IpClaimList(tmp) 281 | *el = tmp2 282 | return nil 283 | } 284 | -------------------------------------------------------------------------------- /doc/ecmp-load-balancing.md: -------------------------------------------------------------------------------- 1 | ECMP Load Balancing for External IPs 2 | ==================================== 3 | 4 | - [ECMP Load Balancing for External IPs](#ecmp-load-balancing-for-external-ips) 5 | * [Introduction](#introduction) 6 | * [Proposed solution](#proposed-solution) 7 | * [Deployment scheme](#deployment-scheme) 8 | + [Diagram](#diagram) 9 | + [Description](#description) 10 | * [Single rack network topology](#single-rack-network-topology) 11 | + [Diagram](#diagram-1) 12 | + [Description](#description-1) 13 | + [Example](#example) 14 | * [Multirack network topology](#multirack-network-topology) 15 | + [Diagram](#diagram-2) 16 | + [Description](#description-2) 17 | + [Example](#example-1) 18 | 19 | ## Introduction 20 | 21 | One of the possible ways to expose k8s services on a bare metal deployments is 22 | using External IPs. Each node runs a kube-proxy process which programs `iptables` 23 | rules to trap access to External IPs and redirect them to the correct backends. 24 | 25 | So in order to access k8s service from the outside we just need to route public 26 | traffic to one of k8s worker nodes which has `kube-proxy` running and thus has 27 | needed `iptables` rules for External IPs. 28 | 29 | ## Proposed solution 30 | 31 | External IP controller is k8s application which is deployed on top of k8s 32 | cluster and which configures External IPs on k8s worker node(s) to provide 33 | IP connectivity. 34 | 35 | This document describes how to expose Kubernetes services using External IP 36 | controller and how to provide load balancing and high availability for External 37 | IPs based on Equal-cost multi-path routing (ECMP). Proposed network topologies 38 | should be considered as examples. 39 | 40 | ## Deployment scheme 41 | 42 | ### Diagram 43 | 44 | ![Deployment scheme](images/ecmp-scheme.png) 45 | 46 | ### Description 47 | 48 | * External IP controller kubernetes application is running on all nodes (or on 49 | a subet of nodes) as a DeamonSet or ReplicaSet with node affinity (up to cloud 50 | operator). 51 | * On start every External IP controller instance pulls information about 52 | services from `kube-api` and brings up all External IPs on `lo` interface on 53 | all target nodes. 54 | * Every External IP controller instance watches `kube-api` for updates in 55 | services with External IPs and: 56 | * When new External IPs appear every instance brings them up on `lo`. 57 | * When service is removed every instance removes appropriate External IPs 58 | from the interface. 59 | * External IP BGP speaker k8s application is running on the same nodes as 60 | External IP controller. It monitors local system and announces External IPs 61 | found on `lo` via BGP. 62 | 63 | ## Single rack network topology 64 | 65 | ### Diagram 66 | 67 | ![Simple network topology](images/bgp-for-ecmp-single.png) 68 | 69 | ### Description 70 | 71 | * There's one instance of Route-Reflector application running in the AS. This 72 | application (k8s PoD) should have static IP which should migrate with it to 73 | the new node in case of fail-over. Otherwise Top of Rack router configuration 74 | should be updated when Route-Reflector IP changes. 75 | * ExIP BGP speakers and Route-Reflector are configured as neighbors within 76 | the same AS. 77 | * Route-Reflector is peered with Top of Rack router within the same AS. 78 | * External IPs assigned by External IP controller on `lo` are exported by 79 | ExIP BGP speakers to the Route-Reflector and thus to Top of Rack router. 80 | * Top of Rack router should have ECMP enabled/configured (for virtual lab and 81 | BIRD routing daemon `merge paths` should be enabled in `kernel` protocol to 82 | support ECMP in routing table). 83 | 84 | ### Example 85 | 86 | Let's see how it works on example: 87 | * Kuberenetes cluster is running on 3 nodes: 88 | ``` 89 | node1 10.210.1.11 90 | node2 10.210.1.12 91 | node3 10.210.1.13 92 | ``` 93 | * External IP controller is running on 2 nodes (node1 and node3), 94 | [example](../examples/simple/externalipcontroller.yaml) with `replicas: 2` and 95 | `HOST_INTERFACE` env variable set to `lo`. 96 | * User creates a service with external IPs ([example](../examples/nginx.yaml)): 97 | ``` 98 | externalIPs: 99 | - 10.0.0.7 100 | - 10.0.0.8 101 | ``` 102 | * External IP controller assigns those IPs to `lo` interface of node1 and 103 | node3: 104 | ``` 105 | 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1 106 | inet 127.0.0.1/8 scope host lo 107 | valid_lft forever preferred_lft forever 108 | inet 10.0.0.8/32 scope global lo 109 | valid_lft forever preferred_lft forever 110 | inet 10.0.0.7/32 scope global lo 111 | valid_lft forever preferred_lft forever 112 | ``` 113 | * ExIP BGP speakers (containers with BIRD) detect changes on node1/node3 and 114 | announce them to peers via BGP. Example of BIRD export filter: 115 | ``` 116 | if ( ifname = "lo" ) then { 117 | if net != 127.0.0.0/8 then accept; 118 | } 119 | ``` 120 | * Top of Rack router (BIRD daemon with `merge paths` enabled) configures the 121 | following routes: 122 | ``` 123 | 10.0.0.7 proto bird 124 | nexthop via 10.210.1.11 dev eth2 weight 1 125 | nexthop via 10.210.1.13 dev eth2 weight 1 126 | 10.0.0.8 proto bird 127 | nexthop via 10.210.1.11 dev eth2 weight 1 128 | nexthop via 10.210.1.13 dev eth2 weight 1 129 | ``` 130 | 131 | ## Multirack network topology 132 | 133 | ### Diagram 134 | 135 | ![Multirack network topology](images/bgp-for-ecmp.png) 136 | 137 | ### Description 138 | 139 | * There's one instance of Route-Reflector application running per rack/AS. This 140 | application (k8s PoD) should have static IP which should migrate with it to 141 | the new node in case of fail-over. Otherwise Top of Rack router configuration 142 | should be updated when Route-Reflector IP changes. 143 | * ExIP BGP speakers and Route-Reflector are configured as neighbors within the 144 | same AS. 145 | See [ExIP BGP speaker configuration example](examples/bird-node1.cfg). 146 | * Route-Reflector is peered with Top of Rack router within the same AS. ECMP 147 | should be configured on both sides (for virtual lab and BIRD routing daemon 148 | `add paths` should be enabled in BGP protocol). 149 | See [Route-Reflector configuration example](examples/bird-rr1.cfg). 150 | * External IPs assigned by External IP controller on `lo` are exported by 151 | ExIP BGP speakers to the Route-Reflector and thus to Top of Rack router. 152 | * Top of Rack router should have ECMP enabled/configured (for virtual lab and 153 | BIRD routing daemon `merge paths` should be enabled in `kernel` protocol to 154 | support ECMP in routing table). 155 | See [Top of Rack router BGP configuration example](examples/bird-tor1.cfg). 156 | * Core router is peered with Top of Rack routers via eBGP. It also has ECMP 157 | enabled/configured (for virtual lab and BIRD routing daemon `merge paths` 158 | should be enabled in `kernel` protocol to support ECMP in routing table). 159 | See [Core router BGP configuration example](examples/bird-core.cfg). 160 | 161 | ### Example 162 | 163 | ![Multirack example topology](images/bgp-for-ecmp-example.png) 164 | 165 | * `nginx` pod and service are running ([example](../examples/nginx.yaml)): 166 | ``` 167 | default nginx-3086523004-0aqgr 1/1 Running 0 1h 10.233.80.3 node-01-003 168 | default nginx-3086523004-3t6hq 1/1 Running 0 2m 10.233.79.67 node-02-002 169 | default nginx-3086523004-ca67h 1/1 Running 0 1h 10.233.85.4 node-01-002 170 | ``` 171 | * Let's check route from `node-01-002` to nginx POD running in rack2: 172 | ``` 173 | $ traceroute -n 10.233.79.67 174 | traceroute to 10.233.79.67 (10.233.79.67), 30 hops max, 60 byte packets 175 | 1 10.211.1.254 0.352 ms 0.316 ms 0.247 ms 176 | 2 192.168.192.5 0.301 ms 0.409 ms 0.353 ms 177 | 3 192.168.192.10 0.459 ms 0.392 ms 0.703 ms 178 | 4 10.211.2.2 0.933 ms 0.879 ms 1.166 ms 179 | 5 10.233.79.67 1.453 ms 1.396 ms 1.246 ms 180 | ``` 181 | * `nginx` service has 2 external IPs (10.0.0.7 and 10.0.0.8). External IP controller 182 | assigned those IPs to `lo` interface on all nodes: 183 | ``` 184 | 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1 185 | inet 127.0.0.1/8 scope host lo 186 | valid_lft forever preferred_lft forever 187 | inet 10.0.0.8/32 scope global lo 188 | valid_lft forever preferred_lft forever 189 | inet 10.0.0.7/32 scope global lo 190 | valid_lft forever preferred_lft forever 191 | ``` 192 | * Routing tables on the routers: 193 | 194 | Core router 195 | ``` 196 | 10.0.0.7 proto bird 197 | nexthop via 192.168.192.6 dev rack01a weight 1 198 | nexthop via 192.168.192.10 dev rack02a weight 1 199 | 10.0.0.8 proto bird 200 | nexthop via 192.168.192.6 dev rack01a weight 1 201 | nexthop via 192.168.192.10 dev rack02a weight 1 202 | ``` 203 | 204 | TOR1 205 | ``` 206 | 10.0.0.7 proto bird 207 | nexthop via 10.211.1.2 dev eth2 weight 1 208 | nexthop via 10.211.1.3 dev eth2 weight 1 209 | 10.0.0.8 proto bird 210 | nexthop via 10.211.1.2 dev eth2 weight 1 211 | nexthop via 10.211.1.3 dev eth2 weight 1 212 | ``` 213 | 214 | TOR2 (no multipath since we have only one node in rack2) 215 | ``` 216 | 10.0.0.7 via 10.211.2.2 dev eth3 proto bird 217 | 10.0.0.8 via 10.211.2.2 dev eth3 proto bird 218 | ``` 219 | -------------------------------------------------------------------------------- /pkg/extensions/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "bytes" 19 | "encoding/json" 20 | 21 | "k8s.io/client-go/1.5/kubernetes" 22 | "k8s.io/client-go/1.5/pkg/api" 23 | "k8s.io/client-go/1.5/pkg/api/unversioned" 24 | "k8s.io/client-go/1.5/pkg/runtime" 25 | "k8s.io/client-go/1.5/pkg/runtime/serializer" 26 | "k8s.io/client-go/1.5/pkg/watch" 27 | "k8s.io/client-go/1.5/rest" 28 | ) 29 | 30 | func WrapClientsetWithExtensions(clientset *kubernetes.Clientset, config *rest.Config) (*WrappedClientset, error) { 31 | restConfig := &rest.Config{} 32 | *restConfig = *config 33 | rest, err := extensionClient(restConfig) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &WrappedClientset{ 38 | Client: rest, 39 | }, nil 40 | } 41 | 42 | func extensionClient(config *rest.Config) (*rest.RESTClient, error) { 43 | config.APIPath = "/apis" 44 | config.ContentConfig = rest.ContentConfig{ 45 | GroupVersion: &unversioned.GroupVersion{ 46 | Group: GroupName, 47 | Version: Version, 48 | }, 49 | NegotiatedSerializer: serializer.DirectCodecFactory{CodecFactory: api.Codecs}, 50 | ContentType: runtime.ContentTypeJSON, 51 | } 52 | return rest.RESTClientFor(config) 53 | } 54 | 55 | type ExtensionsClientset interface { 56 | IPNodes() IPNodesInterface 57 | IPClaims() IPClaimsInterface 58 | IPClaimPools() IPClaimPoolsInterface 59 | } 60 | 61 | type WrappedClientset struct { 62 | Client *rest.RESTClient 63 | } 64 | 65 | type IPClaimsInterface interface { 66 | Create(*IpClaim) (*IpClaim, error) 67 | Get(name string) (*IpClaim, error) 68 | List(api.ListOptions) (*IpClaimList, error) 69 | Watch(api.ListOptions) (watch.Interface, error) 70 | Update(*IpClaim) (*IpClaim, error) 71 | Delete(string, *api.DeleteOptions) error 72 | } 73 | 74 | type IPNodesInterface interface { 75 | Create(*IpNode) (*IpNode, error) 76 | Get(name string) (*IpNode, error) 77 | List(api.ListOptions) (*IpNodeList, error) 78 | Watch(api.ListOptions) (watch.Interface, error) 79 | Update(*IpNode) (*IpNode, error) 80 | Delete(string, *api.DeleteOptions) error 81 | } 82 | 83 | type IPClaimPoolsInterface interface { 84 | Create(*IpClaimPool) (*IpClaimPool, error) 85 | Get(name string) (*IpClaimPool, error) 86 | List(api.ListOptions) (*IpClaimPoolList, error) 87 | Update(*IpClaimPool) (*IpClaimPool, error) 88 | Delete(string, *api.DeleteOptions) error 89 | } 90 | 91 | func (w *WrappedClientset) IPNodes() IPNodesInterface { 92 | return &IPNodesClient{w.Client} 93 | } 94 | 95 | func (w *WrappedClientset) IPClaims() IPClaimsInterface { 96 | return &IpClaimClient{w.Client} 97 | } 98 | 99 | func (w *WrappedClientset) IPClaimPools() IPClaimPoolsInterface { 100 | return &IpClaimPoolClient{w.Client} 101 | } 102 | 103 | type IPNodesClient struct { 104 | client *rest.RESTClient 105 | } 106 | 107 | type IpClaimClient struct { 108 | client *rest.RESTClient 109 | } 110 | 111 | type IpClaimPoolClient struct { 112 | client *rest.RESTClient 113 | } 114 | 115 | func decodeResponseInto(resp []byte, obj interface{}) error { 116 | return json.NewDecoder(bytes.NewReader(resp)).Decode(obj) 117 | } 118 | 119 | func (c *IPNodesClient) Create(ipnode *IpNode) (result *IpNode, err error) { 120 | result = &IpNode{} 121 | resp, err := c.client.Post(). 122 | Namespace("default"). 123 | Resource("ipnodes"). 124 | Body(ipnode). 125 | DoRaw() 126 | if err != nil { 127 | return result, err 128 | } 129 | return result, decodeResponseInto(resp, result) 130 | } 131 | 132 | func (c *IPNodesClient) List(opts api.ListOptions) (result *IpNodeList, err error) { 133 | result = &IpNodeList{} 134 | resp, err := c.client.Get(). 135 | Namespace("default"). 136 | Resource("ipnodes"). 137 | LabelsSelectorParam(opts.LabelSelector). 138 | DoRaw() 139 | if err != nil { 140 | return result, err 141 | } 142 | return result, decodeResponseInto(resp, result) 143 | } 144 | 145 | func (c *IPNodesClient) Watch(opts api.ListOptions) (watch.Interface, error) { 146 | return c.client.Get(). 147 | Namespace("default"). 148 | Prefix("watch"). 149 | Resource("ipnodes"). 150 | VersionedParams(&opts, api.ParameterCodec). 151 | Watch() 152 | } 153 | 154 | func (c *IPNodesClient) Update(ipnode *IpNode) (result *IpNode, err error) { 155 | result = &IpNode{} 156 | resp, err := c.client.Put(). 157 | Namespace("default"). 158 | Resource("ipnodes"). 159 | Name(ipnode.Metadata.Name). 160 | Body(ipnode). 161 | DoRaw() 162 | if err != nil { 163 | return result, err 164 | } 165 | return result, decodeResponseInto(resp, result) 166 | } 167 | 168 | func (c *IPNodesClient) Delete(name string, options *api.DeleteOptions) error { 169 | return c.client.Delete(). 170 | Namespace("default"). 171 | Resource("ipnodes"). 172 | Name(name). 173 | Body(options). 174 | Do(). 175 | Error() 176 | } 177 | 178 | func (c *IPNodesClient) Get(name string) (result *IpNode, err error) { 179 | result = &IpNode{} 180 | resp, err := c.client.Get(). 181 | Namespace("default"). 182 | Resource("ipnodes"). 183 | Name(name). 184 | DoRaw() 185 | if err != nil { 186 | return result, err 187 | } 188 | return result, decodeResponseInto(resp, result) 189 | } 190 | 191 | func (c *IpClaimClient) Get(name string) (result *IpClaim, err error) { 192 | result = &IpClaim{} 193 | err = c.client.Get(). 194 | Namespace("default"). 195 | Resource("ipclaims"). 196 | Name(name). 197 | Do(). 198 | Into(result) 199 | 200 | return result, err 201 | } 202 | 203 | func (c *IpClaimClient) Create(ipclaim *IpClaim) (result *IpClaim, err error) { 204 | result = &IpClaim{} 205 | resp, err := c.client.Post(). 206 | Namespace("default"). 207 | Resource("ipclaims"). 208 | Body(ipclaim). 209 | DoRaw() 210 | if err != nil { 211 | return result, err 212 | } 213 | return result, decodeResponseInto(resp, result) 214 | } 215 | 216 | func (c *IpClaimClient) List(opts api.ListOptions) (result *IpClaimList, err error) { 217 | result = &IpClaimList{} 218 | resp, err := c.client.Get(). 219 | Namespace("default"). 220 | Resource("ipclaims"). 221 | LabelsSelectorParam(opts.LabelSelector). 222 | DoRaw() 223 | if err != nil { 224 | return result, err 225 | } 226 | return result, decodeResponseInto(resp, result) 227 | } 228 | 229 | func (c *IpClaimClient) Watch(opts api.ListOptions) (watch.Interface, error) { 230 | return c.client.Get(). 231 | Namespace("default"). 232 | Prefix("watch"). 233 | Resource("ipclaims"). 234 | Param("resourceVersion", opts.ResourceVersion). 235 | Watch() 236 | } 237 | 238 | func (c *IpClaimClient) Update(ipclaim *IpClaim) (result *IpClaim, err error) { 239 | result = &IpClaim{} 240 | resp, err := c.client.Put(). 241 | Namespace("default"). 242 | Resource("ipclaims"). 243 | Name(ipclaim.Metadata.Name). 244 | Body(ipclaim). 245 | DoRaw() 246 | if err != nil { 247 | return result, err 248 | } 249 | return result, decodeResponseInto(resp, result) 250 | } 251 | 252 | func (c *IpClaimClient) Delete(name string, options *api.DeleteOptions) error { 253 | return c.client.Delete(). 254 | Namespace("default"). 255 | Resource("ipclaims"). 256 | Name(name). 257 | Body(options). 258 | Do(). 259 | Error() 260 | } 261 | 262 | func (c *IpClaimPoolClient) Get(name string) (result *IpClaimPool, err error) { 263 | result = &IpClaimPool{} 264 | err = c.client.Get(). 265 | Namespace("default"). 266 | Resource("ipclaimpools"). 267 | Name(name). 268 | Do(). 269 | Into(result) 270 | 271 | return result, err 272 | } 273 | 274 | func (c *IpClaimPoolClient) Create(ipclaimpool *IpClaimPool) (result *IpClaimPool, err error) { 275 | result = &IpClaimPool{} 276 | resp, err := c.client.Post(). 277 | Namespace("default"). 278 | Resource("ipclaimpools"). 279 | Body(ipclaimpool). 280 | DoRaw() 281 | if err != nil { 282 | return result, err 283 | } 284 | return result, decodeResponseInto(resp, result) 285 | } 286 | 287 | func (c *IpClaimPoolClient) List(opts api.ListOptions) (result *IpClaimPoolList, err error) { 288 | result = &IpClaimPoolList{} 289 | resp, err := c.client.Get(). 290 | Namespace("default"). 291 | Resource("ipclaimpools"). 292 | VersionedParams(&opts, api.ParameterCodec). 293 | DoRaw() 294 | if err != nil { 295 | return result, err 296 | } 297 | return result, decodeResponseInto(resp, result) 298 | } 299 | 300 | func (c *IpClaimPoolClient) Delete(name string, options *api.DeleteOptions) error { 301 | return c.client.Delete(). 302 | Namespace("default"). 303 | Resource("ipclaimpools"). 304 | Name(name). 305 | Body(options). 306 | Do(). 307 | Error() 308 | } 309 | 310 | func (c *IpClaimPoolClient) Update(ipclaimpool *IpClaimPool) (result *IpClaimPool, err error) { 311 | result = &IpClaimPool{} 312 | resp, err := c.client.Put(). 313 | Namespace("default"). 314 | Resource("ipclaimpools"). 315 | Name(ipclaimpool.Metadata.Name). 316 | Body(ipclaimpool). 317 | DoRaw() 318 | if err != nil { 319 | return result, err 320 | } 321 | return result, decodeResponseInto(resp, result) 322 | } 323 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /pkg/scheduler/scheduler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scheduler 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | fclient "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions/testing" 23 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 24 | 25 | "github.com/Mirantis/k8s-externalipcontroller/pkg/utils" 26 | "github.com/stretchr/testify/assert" 27 | "github.com/stretchr/testify/mock" 28 | "k8s.io/client-go/1.5/kubernetes/fake" 29 | "k8s.io/client-go/1.5/pkg/api" 30 | "k8s.io/client-go/1.5/pkg/api/v1" 31 | fcache "k8s.io/client-go/1.5/tools/cache/testing" 32 | "k8s.io/client-go/1.5/pkg/types" 33 | "k8s.io/client-go/1.5/tools/cache" 34 | ) 35 | 36 | func TestServiceWatcher(t *testing.T) { 37 | ext := fclient.NewFakeExtClientset() 38 | lw := fcache.NewFakeControllerSource() 39 | stop := make(chan struct{}) 40 | s := ipClaimScheduler{ 41 | DefaultMask: "24", 42 | serviceSource: lw, 43 | ExtensionsClientset: ext, 44 | changeQueue: workqueue.NewQueue(), 45 | } 46 | 47 | ext.Ipclaimpools.On("List", mock.Anything).Return(&extensions.IpClaimPoolList{}, nil) 48 | 49 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 50 | go s.claimChangeWorker() 51 | go s.serviceWatcher(stop) 52 | defer close(stop) 53 | defer s.changeQueue.Close() 54 | 55 | svc := &v1.Service{ 56 | ObjectMeta: v1.ObjectMeta{Name: "test0"}, 57 | Spec: v1.ServiceSpec{ExternalIPs: []string{"10.10.0.2"}}} 58 | lw.Add(svc) 59 | // let controller process all services 60 | utils.EventualCondition(t, time.Second*1, func() bool { 61 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 62 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 63 | createCall := ext.Ipclaims.Calls[0] 64 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 65 | assert.Equal(t, ipclaim.Spec.Cidr, "10.10.0.2/24", "Unexpected cidr assigned to node") 66 | assert.Equal(t, ipclaim.Metadata.Name, "10-10-0-2-24", "Unexpected name") 67 | 68 | ext.Ipclaims.On("Delete", "10-10-0-2-24", mock.Anything).Return(nil) 69 | lw.Delete(svc) 70 | utils.EventualCondition(t, time.Second*1, func() bool { 71 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 2) 72 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 73 | deleteCall := ext.Ipclaims.Calls[1] 74 | ipclaimName := deleteCall.Arguments[0].(string) 75 | assert.Equal(t, ipclaimName, "10-10-0-2-24", "Unexpected name") 76 | } 77 | 78 | func TestAutoAllocationForServices(t *testing.T) { 79 | ext := fclient.NewFakeExtClientset() 80 | lw := fcache.NewFakeControllerSource() 81 | stop := make(chan struct{}) 82 | svc := v1.Service{ 83 | ObjectMeta: v1.ObjectMeta{ 84 | Name: "need-alloc-svc", 85 | Annotations: map[string]string{"external-ip": "auto"}, 86 | Namespace: api.NamespaceDefault, 87 | }, 88 | } 89 | 90 | fakeClientset := fake.NewSimpleClientset(&v1.ServiceList{Items: []v1.Service{svc}}) 91 | s := ipClaimScheduler{ 92 | DefaultMask: "24", 93 | serviceSource: lw, 94 | ExtensionsClientset: ext, 95 | Clientset: fakeClientset, 96 | changeQueue: workqueue.NewQueue(), 97 | } 98 | 99 | poolCIDR := "192.168.16.248/29" 100 | poolRanges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 101 | pool := &extensions.IpClaimPool{ 102 | Metadata: api.ObjectMeta{Name: "test-pool"}, 103 | Spec: extensions.IpClaimPoolSpec{ 104 | CIDR: poolCIDR, 105 | Ranges: poolRanges, 106 | }, 107 | } 108 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{*pool}} 109 | 110 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 111 | ext.Ipclaimpools.On("Update", mock.Anything).Return(pool, nil) 112 | 113 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 114 | ext.Ipclaims.On("Delete", "192-168-16-250-29", mock.Anything).Return(nil) 115 | ext.Ipclaims.On("Delete", "10-20-0-2-24", mock.Anything).Return(nil) 116 | 117 | go s.claimChangeWorker() 118 | go s.serviceWatcher(stop) 119 | defer close(stop) 120 | defer s.changeQueue.Close() 121 | 122 | lw.Add(&svc) 123 | 124 | utils.EventualCondition(t, time.Second*1, func() bool { 125 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 126 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 127 | 128 | createCall := ext.Ipclaims.Calls[0] 129 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 130 | assert.Equal(t, "192.168.16.250/29", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 131 | assert.Equal(t, "192-168-16-250-29", ipclaim.Metadata.Name, "Unexpected name") 132 | assert.Equal( 133 | t, map[string]string{"ip-pool-name": pool.Metadata.Name}, 134 | ipclaim.Metadata.Labels, 135 | "Labels was not updated by the pool's CIDR") 136 | 137 | poolUpdateCall := ext.Ipclaimpools.Calls[1] 138 | p := poolUpdateCall.Arguments[0].(*extensions.IpClaimPool) 139 | assert.Equal( 140 | t, p.Spec.Allocated, 141 | map[string]string{"192.168.16.250": "192-168-16-250-29"}, 142 | "Allocated was not updated correctly for the pool") 143 | 144 | updatedSvc, _ := fakeClientset.Core().Services(svc.ObjectMeta.Namespace).Get(svc.ObjectMeta.Name) 145 | assert.Equal(t, []string{"192.168.16.250"}, updatedSvc.Spec.ExternalIPs) 146 | 147 | poolList.Items[0].Spec.Allocated = map[string]string{"192.168.16.250": "192-168-16-250-29"} 148 | 149 | changeExternalIPsvc := svc 150 | changeExternalIPsvc.Spec.ExternalIPs = []string{"10.20.0.2"} 151 | delete(changeExternalIPsvc.ObjectMeta.Annotations, "external-ip") 152 | lw.Modify(&changeExternalIPsvc) 153 | 154 | //there should be only 3 calls to Ipclaims at this point: 155 | //1st - creating of 192-168-16-250-29 in AddFunc 156 | //2nd - creating of 10-20-0-2-24 in UpdateFunc 157 | //3rd - deleting of 192-168-16-250-29 in UpdateFunc 158 | //there must not be additional creation of 192-168-16-250-24 - 159 | //ip from the pool but with default mask - at the update of the service 160 | utils.EventualCondition(t, time.Second*1, func() bool { 161 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 3) 162 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 163 | 164 | createCall = ext.Ipclaims.Calls[1] 165 | ipclaim = createCall.Arguments[0].(*extensions.IpClaim) 166 | assert.Equal(t, "10.20.0.2/24", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 167 | assert.Equal(t, "10-20-0-2-24", ipclaim.Metadata.Name, "Unexpected name") 168 | 169 | deleteCall := ext.Ipclaims.Calls[2] 170 | claimName := deleteCall.Arguments[0].(string) 171 | assert.Equal(t, claimName, "192-168-16-250-29", "Unexpected ip claim deleted") 172 | 173 | poolUpdateCall = ext.Ipclaimpools.Calls[4] 174 | p = poolUpdateCall.Arguments[0].(*extensions.IpClaimPool) 175 | assert.NotContains( 176 | t, p.Spec.Allocated, 177 | "192-168-16-250-29", 178 | "Allocated should not contain '192-168-16-250-29'") 179 | } 180 | 181 | func TestClaimNotCreatedIfExternalIPIsAutoAllocated(t *testing.T) { 182 | ext := fclient.NewFakeExtClientset() 183 | lw := fcache.NewFakeControllerSource() 184 | stop := make(chan struct{}) 185 | 186 | poolCIDR := "192.168.16.248/29" 187 | poolRanges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 188 | pool := &extensions.IpClaimPool{ 189 | Metadata: api.ObjectMeta{Name: "test-pool"}, 190 | Spec: extensions.IpClaimPoolSpec{ 191 | CIDR: poolCIDR, 192 | Ranges: poolRanges, 193 | Allocated: map[string]string{"192.168.16.250": "192-168-16-250-29"}, 194 | }, 195 | } 196 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{*pool}} 197 | 198 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 199 | 200 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 201 | 202 | s := ipClaimScheduler{ 203 | DefaultMask: "24", 204 | serviceSource: lw, 205 | ExtensionsClientset: ext, 206 | changeQueue: workqueue.NewQueue(), 207 | } 208 | 209 | go s.claimChangeWorker() 210 | go s.serviceWatcher(stop) 211 | defer close(stop) 212 | defer s.changeQueue.Close() 213 | 214 | svc := v1.Service{ 215 | ObjectMeta: v1.ObjectMeta{ 216 | Name: "need-alloc-svc", 217 | Annotations: map[string]string{"external-ip": "auto"}, 218 | Namespace: api.NamespaceDefault, 219 | }, 220 | Spec: v1.ServiceSpec{ 221 | ExternalIPs: []string{"192.168.16.250", "172.16.0.2"}, 222 | }, 223 | } 224 | lw.Add(&svc) 225 | utils.EventualCondition(t, time.Second*1, func() bool { 226 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 227 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 228 | 229 | createCall := ext.Ipclaims.Calls[0] 230 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 231 | assert.Equal(t, "172.16.0.2/24", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 232 | assert.Equal(t, "172-16-0-2-24", ipclaim.Metadata.Name, "Unexpected name") 233 | } 234 | 235 | func TestAutoAllocatedOnServiceUpdate(t *testing.T) { 236 | ext := fclient.NewFakeExtClientset() 237 | lw := fcache.NewFakeControllerSource() 238 | stop := make(chan struct{}) 239 | 240 | poolCIDR := "192.168.16.248/29" 241 | poolRanges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 242 | pool := &extensions.IpClaimPool{ 243 | Metadata: api.ObjectMeta{Name: "test-pool"}, 244 | Spec: extensions.IpClaimPoolSpec{ 245 | CIDR: poolCIDR, 246 | Ranges: poolRanges, 247 | }, 248 | } 249 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{*pool}} 250 | 251 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 252 | ext.Ipclaimpools.On("Update", mock.Anything).Return(pool, nil) 253 | 254 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 255 | 256 | svc := v1.Service{ 257 | ObjectMeta: v1.ObjectMeta{ 258 | Name: "need-alloc-svc", 259 | Namespace: api.NamespaceDefault, 260 | }, 261 | Spec: v1.ServiceSpec{ 262 | ExternalIPs: []string{"172.16.0.2"}, 263 | }, 264 | } 265 | 266 | fakeClientset := fake.NewSimpleClientset(&v1.ServiceList{Items: []v1.Service{svc}}) 267 | 268 | s := ipClaimScheduler{ 269 | DefaultMask: "24", 270 | serviceSource: lw, 271 | ExtensionsClientset: ext, 272 | Clientset: fakeClientset, 273 | changeQueue: workqueue.NewQueue(), 274 | } 275 | 276 | go s.claimChangeWorker() 277 | go s.serviceWatcher(stop) 278 | defer close(stop) 279 | defer s.changeQueue.Close() 280 | 281 | lw.Add(&svc) 282 | 283 | utils.EventualCondition(t, time.Second*1, func() bool { 284 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 285 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 286 | 287 | annotatedSvc := svc 288 | annotatedSvc.ObjectMeta.Annotations = map[string]string{"external-ip": "auto"} 289 | lw.Modify(&annotatedSvc) 290 | 291 | //one extra create for 172-16-0-2-24 as the service is double processed 292 | utils.EventualCondition(t, time.Second*1, func() bool { 293 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 3) 294 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 295 | 296 | createCall := ext.Ipclaims.Calls[2] 297 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 298 | assert.Equal(t, "192.168.16.250/29", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 299 | assert.Equal(t, "192-168-16-250-29", ipclaim.Metadata.Name, "Unexpected name") 300 | 301 | updatedSvc, _ := fakeClientset.Core().Services(svc.ObjectMeta.Namespace).Get(svc.ObjectMeta.Name) 302 | assert.Contains(t, updatedSvc.Spec.ExternalIPs, "192.168.16.250") 303 | } 304 | 305 | func TestClaimWatcher(t *testing.T) { 306 | ext := fclient.NewFakeExtClientset() 307 | lw := fcache.NewFakeControllerSource() 308 | fss := cache.NewStore(cache.MetaNamespaceKeyFunc) 309 | 310 | svc := v1.Service{ 311 | ObjectMeta: v1.ObjectMeta{ 312 | Name: "some-svc", 313 | Namespace: api.NamespaceDefault, 314 | }, 315 | Spec: v1.ServiceSpec{ 316 | ExternalIPs: []string{"10.10.0.2"}, 317 | }, 318 | } 319 | fss.Add(&svc) 320 | 321 | stop := make(chan struct{}) 322 | s := ipClaimScheduler{ 323 | claimSource: lw, 324 | serviceStore: fss, 325 | ExtensionsClientset: ext, 326 | liveIpNodes: make(map[string]struct{}), 327 | queue: workqueue.NewQueue(), 328 | changeQueue: workqueue.NewQueue(), 329 | } 330 | s.getNode = s.getFairNode 331 | 332 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{}} 333 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 334 | 335 | go s.worker() 336 | go s.claimChangeWorker() 337 | go s.claimWatcher(stop) 338 | defer close(stop) 339 | defer s.queue.Close() 340 | defer s.changeQueue.Close() 341 | 342 | ctrl := false 343 | ownerRef := api.OwnerReference{APIVersion: "v1", Kind: "Service", Name: "some-svc", UID: types.UID("default/some-svc"), Controller: &ctrl} 344 | claim := &extensions.IpClaim{ 345 | Metadata: api.ObjectMeta{Name: "10.10.0.2-24", OwnerReferences: []api.OwnerReference{ownerRef}}, 346 | Spec: extensions.IpClaimSpec{Cidr: "10.10.0.2/24"}, 347 | } 348 | lw.Add(claim) 349 | 350 | ipnodesList := &extensions.IpNodeList{ 351 | Items: []extensions.IpNode{ 352 | { 353 | Metadata: api.ObjectMeta{Name: "first"}, 354 | }, 355 | }, 356 | } 357 | for _, node := range ipnodesList.Items { 358 | s.liveIpNodes[node.Metadata.Name] = struct{}{} 359 | } 360 | 361 | ext.Ipnodes.On("List", mock.Anything).Return(ipnodesList, nil) 362 | ext.Ipclaims.On("Update", mock.Anything).Return(nil) 363 | utils.EventualCondition(t, time.Second*1, func() bool { 364 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 365 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 366 | updatedClaim := ext.Ipclaims.Calls[0].Arguments[0].(*extensions.IpClaim) 367 | assert.Equal(t, updatedClaim.Metadata.Labels, map[string]string{"ipnode": "first"}, 368 | "Labels should be set to scheduled node") 369 | assert.Equal(t, updatedClaim.Spec.NodeName, "first", "NodeName should be set to scheduled node") 370 | } 371 | 372 | func TestMonitorIpNodes(t *testing.T) { 373 | ext := fclient.NewFakeExtClientset() 374 | stop := make(chan struct{}) 375 | ticker := make(chan time.Time, 2) 376 | for i := 0; i < 2; i++ { 377 | ticker <- time.Time{} 378 | } 379 | s := ipClaimScheduler{ 380 | ExtensionsClientset: ext, 381 | liveIpNodes: make(map[string]struct{}), 382 | observedGeneration: make(map[string]int64), 383 | queue: workqueue.NewQueue(), 384 | } 385 | ipnodesList := &extensions.IpNodeList{ 386 | Items: []extensions.IpNode{ 387 | { 388 | Metadata: api.ObjectMeta{ 389 | Name: "first", 390 | }, 391 | Revision: 555, 392 | }, 393 | }, 394 | } 395 | ipclaimsList := &extensions.IpClaimList{ 396 | Items: []extensions.IpClaim{ 397 | { 398 | Metadata: api.ObjectMeta{ 399 | Name: "10.10.0.1-24", 400 | Labels: map[string]string{"ipnode": "first"}, 401 | }, 402 | Spec: extensions.IpClaimSpec{ 403 | Cidr: "10.10.0.1/24", 404 | NodeName: "first", 405 | }, 406 | }, 407 | { 408 | Metadata: api.ObjectMeta{ 409 | Name: "10.10.0.2-24", 410 | Labels: map[string]string{"ipnode": "first"}, 411 | }, 412 | Spec: extensions.IpClaimSpec{ 413 | Cidr: "10.10.0.2/24", 414 | NodeName: "first", 415 | }, 416 | }, 417 | }, 418 | } 419 | ext.Ipnodes.On("List", mock.Anything).Return(ipnodesList, nil).Twice() 420 | ext.Ipclaims.On("List", mock.Anything).Return(ipclaimsList, nil) 421 | go s.monitorIPNodes(stop, ticker) 422 | defer close(stop) 423 | utils.EventualCondition(t, time.Second*1, func() bool { 424 | return assert.ObjectsAreEqual(len(ext.Ipnodes.Calls), 2) 425 | }, "Unexpected call count to ipnodes", ext.Ipnodes.Calls) 426 | utils.EventualCondition(t, time.Second*1, func() bool { 427 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 428 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 429 | assert.Equal(t, s.isLive("first"), false, "first node shouldn't be considered live") 430 | } 431 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 2fb5025c9031b1a9ed4671632becf4d4aa2461ba9c04e210cf21dd3f818838c7 2 | updated: 2016-12-05T18:03:15.669844404+02:00 3 | imports: 4 | - name: cloud.google.com/go 5 | version: 3b1ae45394a234c385be014e9a488f2bb6eef821 6 | subpackages: 7 | - compute/metadata 8 | - internal 9 | - name: github.com/Azure/go-ansiterm 10 | version: 70b2c90b260171e829f1ebd7c17f600c11858dbe 11 | subpackages: 12 | - winterm 13 | - name: github.com/blang/semver 14 | version: 31b736133b98f26d5e078ec9eb591666edfd091f 15 | - name: github.com/coreos/etcd 16 | version: 83347907774bf36cbb261c594a32fd7b0f5dd9f6 17 | subpackages: 18 | - client 19 | - name: github.com/coreos/go-oidc 20 | version: 5644a2f50e2d2d5ba0b474bc5bc55fea1925936d 21 | subpackages: 22 | - http 23 | - jose 24 | - key 25 | - oauth2 26 | - oidc 27 | - name: github.com/coreos/pkg 28 | version: fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 29 | subpackages: 30 | - capnslog 31 | - dlopen 32 | - health 33 | - httputil 34 | - timeutil 35 | - name: github.com/davecgh/go-spew 36 | version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d 37 | subpackages: 38 | - spew 39 | - name: github.com/docker/distribution 40 | version: cd27f179f2c10c5d300e6d09025b538c475b0d51 41 | subpackages: 42 | - digest 43 | - reference 44 | - name: github.com/docker/docker 45 | version: b9f10c951893f9a00865890a5232e85d770c1087 46 | subpackages: 47 | - pkg/jsonlog 48 | - pkg/jsonmessage 49 | - pkg/longpath 50 | - pkg/mount 51 | - pkg/stdcopy 52 | - pkg/symlink 53 | - pkg/system 54 | - pkg/term 55 | - pkg/term/windows 56 | - name: github.com/docker/go-units 57 | version: 0bbddae09c5a5419a8c6dcdd7ff90da3d450393b 58 | - name: github.com/docker/spdystream 59 | version: 449fdfce4d962303d702fec724ef0ad181c92528 60 | subpackages: 61 | - spdy 62 | - name: github.com/emicklei/go-restful 63 | version: 89ef8af493ab468a45a42bb0d89a06fccdd2fb22 64 | subpackages: 65 | - log 66 | - swagger 67 | - name: github.com/ghodss/yaml 68 | version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee 69 | - name: github.com/go-openapi/jsonpointer 70 | version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 71 | - name: github.com/go-openapi/jsonreference 72 | version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272 73 | - name: github.com/go-openapi/spec 74 | version: 6aced65f8501fe1217321abf0749d354824ba2ff 75 | - name: github.com/go-openapi/swag 76 | version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 77 | - name: github.com/gogo/protobuf 78 | version: e18d7aa8f8c624c915db340349aad4c49b10d173 79 | subpackages: 80 | - proto 81 | - sortkeys 82 | - name: github.com/golang/glog 83 | version: 44145f04b68cf362d9c4df2182967c2275eaefed 84 | - name: github.com/golang/groupcache 85 | version: 02826c3e79038b59d737d3b1c0a1d937f71a4433 86 | subpackages: 87 | - lru 88 | - name: github.com/golang/protobuf 89 | version: 8616e8ee5e20a1704615e6c8d7afcdac06087a67 90 | subpackages: 91 | - jsonpb 92 | - proto 93 | - name: github.com/google/gofuzz 94 | version: bbcb9da2d746f8bdbd6a936686a0a6067ada0ec5 95 | - name: github.com/google/gopacket 96 | version: b83f94714c36e30ce851be1d5a0a5226f9f1bca4 97 | subpackages: 98 | - layers 99 | - pcap 100 | - name: github.com/howeyc/gopass 101 | version: 3ca23474a7c7203e0a0a070fd33508f6efdb9b3d 102 | - name: github.com/imdario/mergo 103 | version: 6633656539c1639d9d78127b7d47c622b5d7b6dc 104 | - name: github.com/inconshreveable/mousetrap 105 | version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 106 | - name: github.com/jonboulle/clockwork 107 | version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982 108 | - name: github.com/juju/ratelimit 109 | version: 77ed1c8a01217656d2080ad51981f6e99adaa177 110 | - name: github.com/mailru/easyjson 111 | version: d5b7844b561a7bc640052f1b935f7b800330d7e0 112 | subpackages: 113 | - buffer 114 | - jlexer 115 | - jwriter 116 | - name: github.com/mitchellh/go-wordwrap 117 | version: ad45545899c7b13c020ea92b2072220eefad42b8 118 | - name: github.com/onsi/ginkgo 119 | version: 74c678d97c305753605c338c6c78c49ec104b5e7 120 | subpackages: 121 | - config 122 | - internal/codelocation 123 | - internal/containernode 124 | - internal/failer 125 | - internal/leafnodes 126 | - internal/remote 127 | - internal/spec 128 | - internal/specrunner 129 | - internal/suite 130 | - internal/testingtproxy 131 | - internal/writer 132 | - reporters 133 | - reporters/stenographer 134 | - types 135 | - name: github.com/onsi/gomega 136 | version: a78ae492d53aad5a7a232d0d0462c14c400e3ee7 137 | subpackages: 138 | - format 139 | - internal/assertion 140 | - internal/asyncassertion 141 | - internal/testingtsupport 142 | - matchers 143 | - matchers/support/goraph/bipartitegraph 144 | - matchers/support/goraph/edge 145 | - matchers/support/goraph/node 146 | - matchers/support/goraph/util 147 | - types 148 | - name: github.com/pborman/uuid 149 | version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 150 | - name: github.com/pmezard/go-difflib 151 | version: 792786c7400a136282c1664665ae0a8db921c6c2 152 | subpackages: 153 | - difflib 154 | - name: github.com/PuerkitoBio/purell 155 | version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 156 | - name: github.com/PuerkitoBio/urlesc 157 | version: 5bd2802263f21d8788851d5305584c82a5c75d7e 158 | - name: github.com/Sirupsen/logrus 159 | version: 51fe59aca108dc5680109e7b2051cbdcfa5a253c 160 | - name: github.com/spf13/cobra 161 | version: 9495bc009a56819bdb0ddbc1a373e29c140bc674 162 | - name: github.com/spf13/pflag 163 | version: 5ccb023bc27df288a957c5e994cd44fd19619465 164 | - name: github.com/stretchr/objx 165 | version: 1a9d0bb9f541897e62256577b352fdbc1fb4fd94 166 | - name: github.com/stretchr/testify 167 | version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 168 | subpackages: 169 | - assert 170 | - mock 171 | - name: github.com/ugorji/go 172 | version: f1f1a805ed361a0e078bb537e4ea78cd37dcf065 173 | subpackages: 174 | - codec 175 | - name: github.com/vishvananda/netlink 176 | version: 1e2e08e8a2dcdacaae3f14ac44c5cfa31361f270 177 | subpackages: 178 | - nl 179 | - name: golang.org/x/crypto 180 | version: 1f22c0103821b9390939b6776727195525381532 181 | subpackages: 182 | - bcrypt 183 | - blowfish 184 | - curve25519 185 | - pkcs12 186 | - pkcs12/internal/rc2 187 | - ssh 188 | - ssh/terminal 189 | - name: golang.org/x/net 190 | version: e90d6d0afc4c315a0d87a568ae68577cc15149a0 191 | subpackages: 192 | - context 193 | - context/ctxhttp 194 | - http2 195 | - http2/hpack 196 | - idna 197 | - lex/httplex 198 | - websocket 199 | - name: golang.org/x/oauth2 200 | version: 3c3a985cb79f52a3190fbc056984415ca6763d01 201 | subpackages: 202 | - google 203 | - internal 204 | - jws 205 | - jwt 206 | - name: golang.org/x/sys 207 | version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 208 | subpackages: 209 | - unix 210 | - name: golang.org/x/text 211 | version: 2910a502d2bf9e43193af9d68ca516529614eed3 212 | subpackages: 213 | - cases 214 | - internal/tag 215 | - language 216 | - runes 217 | - secure/bidirule 218 | - secure/precis 219 | - transform 220 | - unicode/bidi 221 | - unicode/norm 222 | - width 223 | - name: google.golang.org/appengine 224 | version: 4f7eeb5305a4ba1966344836ba4af9996b7b4e05 225 | subpackages: 226 | - internal 227 | - internal/app_identity 228 | - internal/base 229 | - internal/datastore 230 | - internal/log 231 | - internal/modules 232 | - internal/remote_api 233 | - internal/urlfetch 234 | - urlfetch 235 | - name: gopkg.in/inf.v0 236 | version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 237 | - name: gopkg.in/yaml.v2 238 | version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 239 | - name: k8s.io/client-go 240 | version: 843f7c4f28b1f647f664f883697107d5c02c5acc 241 | subpackages: 242 | - 1.5/discovery 243 | - 1.5/discovery/fake 244 | - 1.5/kubernetes 245 | - 1.5/kubernetes/fake 246 | - 1.5/kubernetes/typed/apps/v1alpha1 247 | - 1.5/kubernetes/typed/apps/v1alpha1/fake 248 | - 1.5/kubernetes/typed/authentication/v1beta1 249 | - 1.5/kubernetes/typed/authentication/v1beta1/fake 250 | - 1.5/kubernetes/typed/authorization/v1beta1 251 | - 1.5/kubernetes/typed/authorization/v1beta1/fake 252 | - 1.5/kubernetes/typed/autoscaling/v1 253 | - 1.5/kubernetes/typed/autoscaling/v1/fake 254 | - 1.5/kubernetes/typed/batch/v1 255 | - 1.5/kubernetes/typed/batch/v1/fake 256 | - 1.5/kubernetes/typed/certificates/v1alpha1 257 | - 1.5/kubernetes/typed/certificates/v1alpha1/fake 258 | - 1.5/kubernetes/typed/core/v1 259 | - 1.5/kubernetes/typed/core/v1/fake 260 | - 1.5/kubernetes/typed/extensions/v1beta1 261 | - 1.5/kubernetes/typed/extensions/v1beta1/fake 262 | - 1.5/kubernetes/typed/policy/v1alpha1 263 | - 1.5/kubernetes/typed/policy/v1alpha1/fake 264 | - 1.5/kubernetes/typed/rbac/v1alpha1 265 | - 1.5/kubernetes/typed/rbac/v1alpha1/fake 266 | - 1.5/kubernetes/typed/storage/v1beta1 267 | - 1.5/kubernetes/typed/storage/v1beta1/fake 268 | - 1.5/pkg/api 269 | - 1.5/pkg/api/errors 270 | - 1.5/pkg/api/install 271 | - 1.5/pkg/api/meta 272 | - 1.5/pkg/api/meta/metatypes 273 | - 1.5/pkg/api/resource 274 | - 1.5/pkg/api/unversioned 275 | - 1.5/pkg/api/v1 276 | - 1.5/pkg/api/validation/path 277 | - 1.5/pkg/apimachinery 278 | - 1.5/pkg/apimachinery/announced 279 | - 1.5/pkg/apimachinery/registered 280 | - 1.5/pkg/apis/apps 281 | - 1.5/pkg/apis/apps/install 282 | - 1.5/pkg/apis/apps/v1alpha1 283 | - 1.5/pkg/apis/authentication 284 | - 1.5/pkg/apis/authentication/install 285 | - 1.5/pkg/apis/authentication/v1beta1 286 | - 1.5/pkg/apis/authorization 287 | - 1.5/pkg/apis/authorization/install 288 | - 1.5/pkg/apis/authorization/v1beta1 289 | - 1.5/pkg/apis/autoscaling 290 | - 1.5/pkg/apis/autoscaling/install 291 | - 1.5/pkg/apis/autoscaling/v1 292 | - 1.5/pkg/apis/batch 293 | - 1.5/pkg/apis/batch/install 294 | - 1.5/pkg/apis/batch/v1 295 | - 1.5/pkg/apis/batch/v2alpha1 296 | - 1.5/pkg/apis/certificates 297 | - 1.5/pkg/apis/certificates/install 298 | - 1.5/pkg/apis/certificates/v1alpha1 299 | - 1.5/pkg/apis/extensions 300 | - 1.5/pkg/apis/extensions/install 301 | - 1.5/pkg/apis/extensions/v1beta1 302 | - 1.5/pkg/apis/policy 303 | - 1.5/pkg/apis/policy/install 304 | - 1.5/pkg/apis/policy/v1alpha1 305 | - 1.5/pkg/apis/rbac 306 | - 1.5/pkg/apis/rbac/install 307 | - 1.5/pkg/apis/rbac/v1alpha1 308 | - 1.5/pkg/apis/storage 309 | - 1.5/pkg/apis/storage/install 310 | - 1.5/pkg/apis/storage/v1beta1 311 | - 1.5/pkg/auth/user 312 | - 1.5/pkg/conversion 313 | - 1.5/pkg/conversion/queryparams 314 | - 1.5/pkg/fields 315 | - 1.5/pkg/genericapiserver/openapi/common 316 | - 1.5/pkg/labels 317 | - 1.5/pkg/runtime 318 | - 1.5/pkg/runtime/serializer 319 | - 1.5/pkg/runtime/serializer/json 320 | - 1.5/pkg/runtime/serializer/protobuf 321 | - 1.5/pkg/runtime/serializer/recognizer 322 | - 1.5/pkg/runtime/serializer/streaming 323 | - 1.5/pkg/runtime/serializer/versioning 324 | - 1.5/pkg/selection 325 | - 1.5/pkg/third_party/forked/golang/reflect 326 | - 1.5/pkg/types 327 | - 1.5/pkg/util 328 | - 1.5/pkg/util/cert 329 | - 1.5/pkg/util/clock 330 | - 1.5/pkg/util/errors 331 | - 1.5/pkg/util/flowcontrol 332 | - 1.5/pkg/util/framer 333 | - 1.5/pkg/util/homedir 334 | - 1.5/pkg/util/integer 335 | - 1.5/pkg/util/intstr 336 | - 1.5/pkg/util/json 337 | - 1.5/pkg/util/labels 338 | - 1.5/pkg/util/net 339 | - 1.5/pkg/util/parsers 340 | - 1.5/pkg/util/rand 341 | - 1.5/pkg/util/runtime 342 | - 1.5/pkg/util/sets 343 | - 1.5/pkg/util/uuid 344 | - 1.5/pkg/util/validation 345 | - 1.5/pkg/util/validation/field 346 | - 1.5/pkg/util/wait 347 | - 1.5/pkg/util/yaml 348 | - 1.5/pkg/version 349 | - 1.5/pkg/watch 350 | - 1.5/pkg/watch/versioned 351 | - 1.5/plugin/pkg/client/auth 352 | - 1.5/plugin/pkg/client/auth/gcp 353 | - 1.5/plugin/pkg/client/auth/oidc 354 | - 1.5/rest 355 | - 1.5/testing 356 | - 1.5/tools/auth 357 | - 1.5/tools/cache 358 | - 1.5/tools/cache/testing 359 | - 1.5/tools/clientcmd 360 | - 1.5/tools/clientcmd/api 361 | - 1.5/tools/clientcmd/api/latest 362 | - 1.5/tools/clientcmd/api/v1 363 | - 1.5/tools/metrics 364 | - 1.5/transport 365 | - name: k8s.io/kubernetes 366 | version: 42fe4ab0270e44c750d77c682e2fcab394aeb392 367 | subpackages: 368 | - pkg/api 369 | - pkg/api/errors 370 | - pkg/api/install 371 | - pkg/api/meta 372 | - pkg/api/meta/metatypes 373 | - pkg/api/resource 374 | - pkg/api/unversioned 375 | - pkg/api/v1 376 | - pkg/api/validation/path 377 | - pkg/apimachinery 378 | - pkg/apimachinery/announced 379 | - pkg/apimachinery/registered 380 | - pkg/apis/apps 381 | - pkg/apis/apps/install 382 | - pkg/apis/apps/v1beta1 383 | - pkg/apis/authentication 384 | - pkg/apis/authentication/install 385 | - pkg/apis/authentication/v1beta1 386 | - pkg/apis/authorization 387 | - pkg/apis/authorization/install 388 | - pkg/apis/authorization/v1beta1 389 | - pkg/apis/autoscaling 390 | - pkg/apis/autoscaling/install 391 | - pkg/apis/autoscaling/v1 392 | - pkg/apis/batch 393 | - pkg/apis/batch/install 394 | - pkg/apis/batch/v1 395 | - pkg/apis/batch/v2alpha1 396 | - pkg/apis/certificates 397 | - pkg/apis/certificates/install 398 | - pkg/apis/certificates/v1alpha1 399 | - pkg/apis/componentconfig 400 | - pkg/apis/componentconfig/install 401 | - pkg/apis/componentconfig/v1alpha1 402 | - pkg/apis/extensions 403 | - pkg/apis/extensions/install 404 | - pkg/apis/extensions/v1beta1 405 | - pkg/apis/policy 406 | - pkg/apis/policy/install 407 | - pkg/apis/policy/v1beta1 408 | - pkg/apis/rbac 409 | - pkg/apis/rbac/install 410 | - pkg/apis/rbac/v1alpha1 411 | - pkg/apis/storage 412 | - pkg/apis/storage/install 413 | - pkg/apis/storage/v1beta1 414 | - pkg/auth/user 415 | - pkg/client/clientset_generated/internalclientset 416 | - pkg/client/clientset_generated/internalclientset/typed/apps/internalversion 417 | - pkg/client/clientset_generated/internalclientset/typed/authentication/internalversion 418 | - pkg/client/clientset_generated/internalclientset/typed/authorization/internalversion 419 | - pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion 420 | - pkg/client/clientset_generated/internalclientset/typed/batch/internalversion 421 | - pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion 422 | - pkg/client/clientset_generated/internalclientset/typed/core/internalversion 423 | - pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion 424 | - pkg/client/clientset_generated/internalclientset/typed/policy/internalversion 425 | - pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion 426 | - pkg/client/clientset_generated/internalclientset/typed/storage/internalversion 427 | - pkg/client/leaderelection 428 | - pkg/client/leaderelection/resourcelock 429 | - pkg/client/metrics 430 | - pkg/client/record 431 | - pkg/client/restclient 432 | - pkg/client/transport 433 | - pkg/client/typed/discovery 434 | - pkg/client/unversioned/clientcmd/api 435 | - pkg/client/unversioned/remotecommand 436 | - pkg/conversion 437 | - pkg/conversion/queryparams 438 | - pkg/fields 439 | - pkg/genericapiserver/openapi/common 440 | - pkg/httplog 441 | - pkg/kubelet/qos 442 | - pkg/kubelet/server/remotecommand 443 | - pkg/kubelet/types 444 | - pkg/labels 445 | - pkg/master/ports 446 | - pkg/runtime 447 | - pkg/runtime/serializer 448 | - pkg/runtime/serializer/json 449 | - pkg/runtime/serializer/protobuf 450 | - pkg/runtime/serializer/recognizer 451 | - pkg/runtime/serializer/streaming 452 | - pkg/runtime/serializer/versioning 453 | - pkg/selection 454 | - pkg/types 455 | - pkg/util 456 | - pkg/util/cert 457 | - pkg/util/clock 458 | - pkg/util/config 459 | - pkg/util/errors 460 | - pkg/util/exec 461 | - pkg/util/flag 462 | - pkg/util/flowcontrol 463 | - pkg/util/framer 464 | - pkg/util/httpstream 465 | - pkg/util/httpstream/spdy 466 | - pkg/util/integer 467 | - pkg/util/interrupt 468 | - pkg/util/intstr 469 | - pkg/util/json 470 | - pkg/util/labels 471 | - pkg/util/net 472 | - pkg/util/parsers 473 | - pkg/util/rand 474 | - pkg/util/runtime 475 | - pkg/util/sets 476 | - pkg/util/strategicpatch 477 | - pkg/util/term 478 | - pkg/util/uuid 479 | - pkg/util/validation 480 | - pkg/util/validation/field 481 | - pkg/util/wait 482 | - pkg/util/wsstream 483 | - pkg/util/yaml 484 | - pkg/version 485 | - pkg/watch 486 | - pkg/watch/versioned 487 | - plugin/pkg/client/auth 488 | - plugin/pkg/client/auth/gcp 489 | - plugin/pkg/client/auth/oidc 490 | - third_party/forked/golang/json 491 | - third_party/forked/golang/netutil 492 | - third_party/forked/golang/reflect 493 | testImports: [] 494 | -------------------------------------------------------------------------------- /pkg/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scheduler 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "strings" 21 | "sync" 22 | "time" 23 | 24 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 25 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 26 | 27 | "github.com/golang/glog" 28 | "k8s.io/client-go/1.5/kubernetes" 29 | "k8s.io/client-go/1.5/pkg/api" 30 | apierrors "k8s.io/client-go/1.5/pkg/api/errors" 31 | "k8s.io/client-go/1.5/pkg/api/v1" 32 | "k8s.io/client-go/1.5/pkg/fields" 33 | "k8s.io/client-go/1.5/pkg/labels" 34 | "k8s.io/client-go/1.5/pkg/runtime" 35 | "k8s.io/client-go/1.5/pkg/watch" 36 | "k8s.io/client-go/1.5/rest" 37 | "k8s.io/client-go/1.5/tools/cache" 38 | "k8s.io/client-go/1.5/pkg/types" 39 | ) 40 | 41 | const ( 42 | AutoExternalAnnotationKey = "external-ip" 43 | AutoExternalAnnotationValue = "auto" 44 | ) 45 | 46 | func NewIPClaimScheduler(config *rest.Config, mask string, monitorInterval time.Duration, nodeFilter string) (*ipClaimScheduler, error) { 47 | clientset, err := kubernetes.NewForConfig(config) 48 | if err != nil { 49 | return nil, err 50 | } 51 | ext, err := extensions.WrapClientsetWithExtensions(clientset, config) 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | serviceSource := &cache.ListWatch{ 57 | ListFunc: func(options api.ListOptions) (runtime.Object, error) { 58 | return clientset.Core().Services(api.NamespaceAll).List(options) 59 | }, 60 | WatchFunc: func(options api.ListOptions) (watch.Interface, error) { 61 | return clientset.Core().Services(api.NamespaceAll).Watch(options) 62 | }, 63 | } 64 | 65 | claimSource := cache.NewListWatchFromClient(ext.Client, "ipclaims", api.NamespaceAll, fields.Everything()) 66 | scheduler := ipClaimScheduler{ 67 | Config: config, 68 | Clientset: clientset, 69 | ExtensionsClientset: ext, 70 | DefaultMask: mask, 71 | 72 | monitorPeriod: monitorInterval, 73 | serviceSource: serviceSource, 74 | claimSource: claimSource, 75 | 76 | observedGeneration: make(map[string]int64), 77 | liveIpNodes: make(map[string]struct{}), 78 | 79 | queue: workqueue.NewQueue(), 80 | changeQueue: workqueue.NewQueue(), 81 | } 82 | 83 | switch nodeFilter{ 84 | case "fair": 85 | scheduler.getNode = scheduler.getFairNode 86 | case "first-alive": 87 | scheduler.getNode = scheduler.getFirstAliveNode 88 | default: 89 | return nil, errors.New("Incorrect node filter is provided") 90 | } 91 | 92 | return &scheduler, nil 93 | } 94 | 95 | type nodeFilter func([]*extensions.IpNode) *extensions.IpNode 96 | 97 | type ipClaimScheduler struct { 98 | Config *rest.Config 99 | Clientset kubernetes.Interface 100 | ExtensionsClientset extensions.ExtensionsClientset 101 | DefaultMask string 102 | 103 | serviceSource cache.ListerWatcher 104 | claimSource cache.ListerWatcher 105 | 106 | monitorPeriod time.Duration 107 | observedGeneration map[string]int64 108 | liveSync sync.Mutex 109 | liveIpNodes map[string]struct{} 110 | 111 | claimStore cache.Store 112 | serviceStore cache.Store 113 | 114 | getNode nodeFilter 115 | 116 | queue workqueue.QueueType 117 | changeQueue workqueue.QueueType 118 | } 119 | 120 | func (s *ipClaimScheduler) Run(stop chan struct{}) { 121 | glog.V(3).Infof("Starting monitor goroutine.") 122 | go s.monitorIPNodes(stop, time.Tick(s.monitorPeriod)) 123 | // let's give controllers some time to register themselves after scheduler restart 124 | // TODO(dshulyak) consider to run monitor both for leaders/non-leaders 125 | time.Sleep(s.monitorPeriod) 126 | glog.V(3).Infof("Starting all other worker goroutines.") 127 | go s.worker() 128 | go s.claimChangeWorker() 129 | go s.serviceWatcher(stop) 130 | go s.claimWatcher(stop) 131 | <-stop 132 | s.queue.Close() 133 | s.changeQueue.Close() 134 | } 135 | 136 | // serviceWatcher creates/deletes IPClaim based on requirements from 137 | // service 138 | func (s *ipClaimScheduler) serviceWatcher(stop chan struct{}) { 139 | store, controller := cache.NewInformer( 140 | s.serviceSource, 141 | &v1.Service{}, 142 | 0, 143 | cache.ResourceEventHandlerFuncs{ 144 | AddFunc: func(obj interface{}) { 145 | svc := obj.(*v1.Service) 146 | s.processExternalIPs(svc) 147 | }, 148 | UpdateFunc: func(old, cur interface{}) { 149 | curSvc := cur.(*v1.Service) 150 | s.processExternalIPs(curSvc) 151 | 152 | oldSvc := old.(*v1.Service) 153 | s.processOldService(oldSvc) 154 | }, 155 | DeleteFunc: func(obj interface{}) { 156 | svc := obj.(*v1.Service) 157 | s.processOldService(svc) 158 | }, 159 | }, 160 | ) 161 | s.serviceStore = store 162 | controller.Run(stop) 163 | } 164 | 165 | // we must take into account that loss of events and double processing of 166 | // service objects might occur (due the way the object cache works); thus 167 | // auto allocation must be done only in case a service is properly annotated 168 | // and there is no already auto allocated IP for it 169 | func (s *ipClaimScheduler) processExternalIPs(svc *v1.Service) { 170 | foundAuto := false 171 | 172 | pools, err := s.ExtensionsClientset.IPClaimPools().List(api.ListOptions{}) 173 | if err != nil { 174 | glog.Errorf("Error retrieving list of IP pools. Details: %v", err) 175 | } 176 | 177 | for _, ip := range svc.Spec.ExternalIPs { 178 | if p := poolByAllocatedIP(ip, pools); p != nil { 179 | foundAuto = true 180 | continue 181 | } 182 | s.addClaimChangeRequest(makeIPClaim(ip, s.DefaultMask, svc), cache.Added) 183 | } 184 | 185 | if annotated := checkAnnotation(svc); annotated && !foundAuto { 186 | s.autoAllocateExternalIP(svc, pools) 187 | } 188 | } 189 | 190 | func poolByAllocatedIP(ip string, poolList *extensions.IpClaimPoolList) *extensions.IpClaimPool { 191 | for _, pool := range poolList.Items { 192 | if _, exists := pool.Spec.Allocated[ip]; exists { 193 | return &pool 194 | } 195 | } 196 | return nil 197 | } 198 | 199 | func (s *ipClaimScheduler) autoAllocateExternalIP(svc *v1.Service, poolList *extensions.IpClaimPoolList) { 200 | glog.V(5).Infof("Try to auto allocate external IP for service '%v'", svc.ObjectMeta.Name) 201 | 202 | var freeIP string 203 | var pool extensions.IpClaimPool 204 | 205 | for _, p := range poolList.Items { 206 | ip, err := p.AvailableIP() 207 | if err != nil { 208 | glog.Errorf( 209 | "Fail to retrieve free IP from the pool '%v'; skipping to a next one. Details: %v", 210 | p.Metadata.Name, err) 211 | continue 212 | } 213 | freeIP = ip 214 | pool = p 215 | break 216 | } 217 | 218 | if len(freeIP) == 0 { 219 | glog.Errorf( 220 | "Fail to provide external IP for service '%v'. All pools are exhausted.", 221 | svc.ObjectMeta.Name) 222 | return 223 | } 224 | 225 | mask := strings.Split(pool.Spec.CIDR, "/")[1] 226 | 227 | ipclaim := makeIPClaim(freeIP, mask, svc) 228 | ipclaim.Metadata.SetLabels( 229 | map[string]string{"ip-pool-name": pool.Metadata.Name}) 230 | 231 | s.addClaimChangeRequest(ipclaim, cache.Added) 232 | 233 | err := updatePoolAllocation(s.ExtensionsClientset, &pool, freeIP, ipclaim.Metadata.Name) 234 | if err != nil { 235 | glog.Errorf("Unable to update IP pool's '%v' allocation. Details: %v", 236 | pool.Metadata.Name, err) 237 | } 238 | 239 | err = addServiceExternalIP(svc, s.Clientset, freeIP) 240 | if err != nil { 241 | glog.Errorf("Unable to update ExternalIPs for service '%v'. Details: %v", 242 | svc.ObjectMeta, err) 243 | } 244 | } 245 | 246 | func (s *ipClaimScheduler) claimWatcher(stop chan struct{}) { 247 | store, controller := cache.NewInformer( 248 | s.claimSource, 249 | &extensions.IpClaim{}, 250 | 10 * time.Second, 251 | cache.ResourceEventHandlerFuncs{ 252 | AddFunc: func(obj interface{}) { 253 | claim := obj.(*extensions.IpClaim) 254 | glog.V(3).Infof("IP claim '%v' was created", claim.Metadata.Name) 255 | key, err := cache.MetaNamespaceKeyFunc(claim) 256 | if err != nil { 257 | glog.Errorf("Error getting key for IP claim: %v", err) 258 | } 259 | s.queue.Add(key) 260 | }, 261 | UpdateFunc: func(_, cur interface{}) { 262 | claim := cur.(*extensions.IpClaim) 263 | glog.V(3).Infof("IP claim '%v' was updated. Resource version: %v", 264 | claim.Metadata.Name, claim.Metadata.ResourceVersion) 265 | key, err := cache.MetaNamespaceKeyFunc(claim) 266 | if err != nil { 267 | glog.Errorf("Error getting key for IP claim: %v", err) 268 | } 269 | s.queue.Add(key) 270 | }, 271 | DeleteFunc: func(obj interface{}) { 272 | claim := obj.(*extensions.IpClaim) 273 | glog.V(3).Infof("IP claim '%v' was deleted. Resource version: %v", 274 | claim.Metadata.Name, claim.Metadata.ResourceVersion) 275 | }, 276 | }, 277 | ) 278 | s.claimStore = store 279 | controller.Run(stop) 280 | } 281 | 282 | func (s *ipClaimScheduler) findAliveNodes(ipnodes []extensions.IpNode) (result []*extensions.IpNode) { 283 | s.liveSync.Lock() 284 | s.liveSync.Unlock() 285 | for i := range ipnodes { 286 | node := ipnodes[i] 287 | if _, ok := s.liveIpNodes[node.Metadata.Name]; ok { 288 | result = append(result, &node) 289 | } 290 | } 291 | return result 292 | } 293 | 294 | func (s *ipClaimScheduler) worker() { 295 | glog.V(1).Infof("Starting worker to process IP claims") 296 | for { 297 | key, quit := s.queue.Get() 298 | glog.V(3).Infof("Got IP claim '%v' to process", key) 299 | if quit { 300 | return 301 | } 302 | item, exists, _ := s.claimStore.GetByKey(key.(string)) 303 | if exists { 304 | err := s.processIpClaim(item.(*extensions.IpClaim)) 305 | if err != nil { 306 | glog.Errorf("Error processing IP claim: %v", err) 307 | s.queue.Add(key) 308 | } 309 | } 310 | glog.V(5).Infof("Processing of IP claim '%v' was completed", key) 311 | s.queue.Done(key) 312 | } 313 | } 314 | 315 | func (s *ipClaimScheduler) claimChangeWorker() { 316 | client := s.ExtensionsClientset.IPClaims() 317 | for { 318 | req, quit := s.changeQueue.Get() 319 | glog.V(3).Infof("Got IP claim change request '%v' to process", req) 320 | if quit { 321 | return 322 | } 323 | changeReq := req.(*cache.Delta) 324 | claim := changeReq.Object.(*extensions.IpClaim) 325 | switch changeReq.Type { 326 | case cache.Added: 327 | _, err := client.Create(claim) 328 | if apierrors.IsAlreadyExists(err) { 329 | // Let's add new owner ref to the owner ref list of the existing IP claim 330 | glog.V(3).Infof("IP claim '%v' exists already", claim.Metadata.Name) 331 | existing, err := client.Get(claim.Metadata.Name) 332 | if err != nil { 333 | glog.Errorf("Unable to get IP claim '%v'. Details: %v", claim.Metadata.Name, err) 334 | s.changeQueue.Add(changeReq) 335 | } 336 | newOwnerRef := claim.Metadata.OwnerReferences[0] 337 | existOwnerRefs := existing.Metadata.OwnerReferences 338 | alreadyThere := false 339 | for r := range existOwnerRefs { 340 | if newOwnerRef.UID == existOwnerRefs[r].UID { 341 | glog.V(5).Infof("Service '%v' is referenced in IP claim '%v' already", 342 | newOwnerRef.UID, claim.Metadata.Name) 343 | alreadyThere = true 344 | break 345 | } 346 | } 347 | if !alreadyThere { 348 | existing.Metadata.OwnerReferences = append(existOwnerRefs, newOwnerRef) 349 | s.addClaimChangeRequest(existing, cache.Updated) 350 | glog.V(3).Infof("IP claim '%v' is to be updated with reference to service '%v'", 351 | claim.Metadata.Name, newOwnerRef.UID) 352 | } 353 | } else if err == nil { 354 | glog.V(3).Infof("IP claim '%v' was created", claim.Metadata.Name) 355 | } else { 356 | glog.Errorf("Unable to create IP claim '%v'. Details: %v", claim.Metadata.Name, err) 357 | } 358 | case cache.Updated: 359 | _, err := client.Update(claim) 360 | if err == nil { 361 | glog.V(3).Infof("IP claim '%v' was updated with node '%v'. Resource version: %v", 362 | claim.Metadata.Name, claim.Spec.NodeName, claim.Metadata.ResourceVersion) 363 | } else { 364 | glog.Errorf("Unable to update IP claim '%v'. Details: %v", claim.Metadata.Name, err) 365 | } 366 | case cache.Deleted: 367 | err := client.Delete(claim.Metadata.Name, &api.DeleteOptions{}) 368 | if err != nil { 369 | glog.Errorf("Unable to delete IP claim '%v'. Details: %v", claim.Metadata.Name, err) 370 | } 371 | } 372 | glog.V(3).Infof("Processing of IP claim '%v' change request was completed", claim.Metadata.Name) 373 | s.changeQueue.Done(req) 374 | } 375 | } 376 | 377 | func (s *ipClaimScheduler) addClaimChangeRequest(claim *extensions.IpClaim, change cache.DeltaType) { 378 | req := &cache.Delta{ 379 | Object: claim, 380 | Type: change, 381 | } 382 | s.changeQueue.Add(req) 383 | } 384 | 385 | func (s *ipClaimScheduler) processOldService(svc *v1.Service) { 386 | refs := map[string]struct{}{} 387 | svcList := s.serviceStore.List() 388 | for i := range svcList { 389 | svc := svcList[i].(*v1.Service) 390 | for _, ip := range svc.Spec.ExternalIPs { 391 | refs[ip] = struct{}{} 392 | } 393 | } 394 | 395 | pools := s.getIPClaimPoolList() 396 | for _, ip := range svc.Spec.ExternalIPs { 397 | if _, ok := refs[ip]; !ok { 398 | s.deleteIPClaimAndAllocation(ip, pools) 399 | } 400 | } 401 | } 402 | 403 | func (s *ipClaimScheduler) getIPClaimByIP(ip string, pools *extensions.IpClaimPoolList) (*extensions.IpClaim, error) { 404 | mask := "" 405 | if p := poolByAllocatedIP(ip, pools); p != nil { 406 | mask = strings.Split(p.Spec.CIDR, "/")[1] 407 | } else { 408 | mask = s.DefaultMask 409 | } 410 | ipParts := strings.Split(ip, ".") 411 | key := strings.Join([]string{strings.Join(ipParts, "-"), mask}, "-") 412 | return s.ExtensionsClientset.IPClaims().Get(key) 413 | } 414 | 415 | func (s *ipClaimScheduler) getIPClaimPoolList() *extensions.IpClaimPoolList { 416 | pools, err := s.ExtensionsClientset.IPClaimPools().List(api.ListOptions{}) 417 | if err != nil { 418 | glog.Errorf("Error retrieving list of IP pools. Details: %v", err) 419 | } 420 | return pools 421 | } 422 | 423 | func (s *ipClaimScheduler) deleteIPClaimAndAllocation(ip string, pools *extensions.IpClaimPoolList) { 424 | if p := poolByAllocatedIP(ip, pools); p != nil { 425 | s.addClaimChangeRequest(makeIPClaim(ip, strings.Split(p.Spec.CIDR, "/")[1], nil), cache.Deleted) 426 | delete(p.Spec.Allocated, ip) 427 | 428 | glog.V(2).Infof("Try to update IP pool with object %v", p) 429 | _, err := s.ExtensionsClientset.IPClaimPools().Update(p) 430 | if err != nil { 431 | glog.Errorf("Unable to update IP pool '%v'. Details: %v", p.Metadata.Name, err) 432 | } 433 | } else { 434 | s.addClaimChangeRequest(makeIPClaim(ip, s.DefaultMask, nil), cache.Deleted) 435 | } 436 | } 437 | 438 | // returns list of owner references that are relevant at the moment 439 | func (s *ipClaimScheduler) ownersAlive(claim *extensions.IpClaim) []api.OwnerReference { 440 | // only services can be the claim owners for now 441 | owners := []api.OwnerReference{} 442 | for _, owner := range claim.Metadata.OwnerReferences { 443 | _, exists, err := s.serviceStore.GetByKey(string(owner.UID)) 444 | if err != nil { 445 | glog.Errorf("Checking claim '%v' owners: error getting service '%v' from cache: %v", claim.Metadata.Name, owner.UID, err) 446 | } 447 | if !exists { 448 | glog.V(5).Infof("Checking claim '%v' owners: service '%v' is not in cache", claim.Metadata.Name, owner.UID) 449 | ns_name := strings.Split(string(owner.UID), "/") 450 | // "an empty namespace may not be set when a resource name is provided" error is thrown when 451 | // calling Services.Get w/o a namespace 452 | if len(ns_name) == 2 { 453 | _, err = s.Clientset.Core().Services(ns_name[0]).Get(ns_name[1]) 454 | } else { 455 | glog.Errorf("Checking claim '%v' owners: cannot get namespace for service '%v'", claim.Metadata.Name, owner.UID) 456 | } 457 | if apierrors.IsNotFound(err) { 458 | glog.V(5).Infof("Checking claim '%v' owners: service '%v' does not exist", claim.Metadata.Name, owner.UID) 459 | continue 460 | } 461 | if err != nil { 462 | glog.Errorf("Checking claim '%v' owners: service '%v' get error: %v", claim.Metadata.Name, owner.UID, err) 463 | } 464 | } 465 | owners = append(owners, owner) 466 | } 467 | glog.V(5).Infof("Checking claim '%v' owners: %v", claim.Metadata.Name, owners) 468 | return owners 469 | } 470 | 471 | func (s *ipClaimScheduler) processIpClaim(claim *extensions.IpClaim) error { 472 | ownersAlive := s.ownersAlive(claim) 473 | if len(ownersAlive) == 0 { 474 | // all owner links are irrelevant 475 | pools := s.getIPClaimPoolList() 476 | s.deleteIPClaimAndAllocation(strings.Split(claim.Spec.Cidr, "/")[0], pools) 477 | return nil 478 | } else if len(ownersAlive) < len(claim.Metadata.OwnerReferences) { 479 | // some owner links are irrelevant 480 | claim.Metadata.OwnerReferences = ownersAlive 481 | s.addClaimChangeRequest(claim, cache.Updated) 482 | } 483 | 484 | if claim.Spec.NodeName != "" && s.isLive(claim.Spec.NodeName) { 485 | return nil 486 | } 487 | ipnodes, err := s.ExtensionsClientset.IPNodes().List(api.ListOptions{}) 488 | if err != nil { 489 | return err 490 | } 491 | // this needs to be queued and requeued in case of node absence 492 | if len(ipnodes.Items) == 0 { 493 | return fmt.Errorf("No nodes") 494 | } 495 | liveNodes := s.findAliveNodes(ipnodes.Items) 496 | if len(liveNodes) == 0 { 497 | return fmt.Errorf("No live nodes") 498 | } 499 | ipnode := s.getNode(liveNodes) 500 | claim.Metadata.SetLabels(map[string]string{"ipnode": ipnode.Metadata.Name}) 501 | claim.Spec.NodeName = ipnode.Metadata.Name 502 | glog.V(3).Infof("Scheduling IP claim '%v' on a node '%v'", 503 | claim.Metadata.Name, claim.Spec.NodeName) 504 | s.addClaimChangeRequest(claim, cache.Updated) 505 | return nil 506 | } 507 | 508 | func (s *ipClaimScheduler) monitorIPNodes(stop chan struct{}, ticker <-chan time.Time) { 509 | for { 510 | select { 511 | case <-stop: 512 | return 513 | case <-ticker: 514 | ipnodes, err := s.ExtensionsClientset.IPNodes().List(api.ListOptions{}) 515 | if err != nil { 516 | glog.Errorf("Error getting IP nodes: %v", err) 517 | } 518 | 519 | for _, ipnode := range ipnodes.Items { 520 | name := ipnode.Metadata.Name 521 | version := s.observedGeneration[name] 522 | curVersion := ipnode.Revision 523 | if version < curVersion { 524 | s.observedGeneration[name] = curVersion 525 | s.liveSync.Lock() 526 | glog.V(3).Infof("IP node '%v' is alive. Versions: %v - %v", 527 | name, version, curVersion) 528 | s.liveIpNodes[name] = struct{}{} 529 | s.liveSync.Unlock() 530 | } else { 531 | s.liveSync.Lock() 532 | glog.V(3).Infof("IP node '%v' is dead. Versions: %v - %v", 533 | name, version, curVersion) 534 | delete(s.liveIpNodes, name) 535 | s.liveSync.Unlock() 536 | labelSelector := labels.Set(map[string]string{"ipnode": name}) 537 | ipclaims, err := s.ExtensionsClientset.IPClaims().List( 538 | api.ListOptions{ 539 | LabelSelector: labelSelector.AsSelector(), 540 | }, 541 | ) 542 | if err != nil { 543 | glog.Errorf("Error fetching list of IP claims: %v", err) 544 | break 545 | } 546 | for _, ipclaim := range ipclaims.Items { 547 | // TODO don't update - send to queue for rescheduling instead 548 | glog.Infof("Sending IP claim '%v' for rescheduling. CIDR '%v', previous node '%v'", 549 | ipclaim.Metadata.Name, ipclaim.Spec.Cidr, ipclaim.Spec.NodeName) 550 | key, err := cache.MetaNamespaceKeyFunc(&ipclaim) 551 | if err != nil { 552 | glog.Errorf("Error getting key for IP claim: %v", err) 553 | } else { 554 | s.queue.Add(key) 555 | } 556 | } 557 | } 558 | } 559 | } 560 | } 561 | } 562 | 563 | func (s *ipClaimScheduler) isLive(name string) bool { 564 | s.liveSync.Lock() 565 | defer s.liveSync.Unlock() 566 | _, ok := s.liveIpNodes[name] 567 | return ok 568 | } 569 | 570 | func checkAnnotation(svc *v1.Service) bool { 571 | if svc.ObjectMeta.Annotations != nil { 572 | val, exists := svc.ObjectMeta.Annotations[AutoExternalAnnotationKey] 573 | if exists { 574 | glog.V(5).Infof( 575 | "Auto-allocation annotation (key '%v') is provided for service '%v' with value '%v'", 576 | AutoExternalAnnotationKey, svc.ObjectMeta.Name, val, 577 | ) 578 | if val == AutoExternalAnnotationValue { 579 | return true 580 | } 581 | glog.Warning("Only value '%v' is processed for annotation key '%v'", 582 | AutoExternalAnnotationValue, AutoExternalAnnotationKey) 583 | } 584 | } 585 | return false 586 | } 587 | 588 | func updatePoolAllocation(ext extensions.ExtensionsClientset, pool *extensions.IpClaimPool, ip, claimName string) error { 589 | if pool.Spec.Allocated != nil { 590 | pool.Spec.Allocated[ip] = claimName 591 | } else { 592 | pool.Spec.Allocated = map[string]string{ip: claimName} 593 | } 594 | 595 | glog.V(2).Infof("Update IP pool with object %v", pool) 596 | _, err := ext.IPClaimPools().Update(pool) 597 | return err 598 | } 599 | 600 | func addServiceExternalIP(svc *v1.Service, kcs kubernetes.Interface, ip string) error { 601 | glog.V(5).Infof( 602 | "Try to update externalIPs list of service '%v' with IP address '%v'", 603 | svc.ObjectMeta.Name, ip, 604 | ) 605 | svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip) 606 | _, err := kcs.Core().Services(svc.ObjectMeta.Namespace).Update(svc) 607 | return err 608 | } 609 | 610 | func makeIPClaim(ip, mask string, svc *v1.Service) *extensions.IpClaim { 611 | ipParts := strings.Split(ip, ".") 612 | key := strings.Join([]string{strings.Join(ipParts, "-"), mask}, "-") 613 | cidr := strings.Join([]string{ip, mask}, "/") 614 | 615 | glog.V(2).Infof("Creating IP claim '%v'", key) 616 | 617 | meta := api.ObjectMeta{Name: key} 618 | if svc != nil { 619 | svc_key, err := cache.MetaNamespaceKeyFunc(svc) 620 | if err != nil { 621 | return nil 622 | } 623 | ctrl := false 624 | ownerRef := api.OwnerReference{APIVersion: "v1", Kind: "Service", Name: svc.Name, UID: types.UID(svc_key), Controller: &ctrl} 625 | meta.OwnerReferences = []api.OwnerReference{ownerRef} 626 | } 627 | 628 | ipclaim := &extensions.IpClaim{ 629 | Metadata: meta, 630 | Spec: extensions.IpClaimSpec{Cidr: cidr}, 631 | } 632 | return ipclaim 633 | } 634 | --------------------------------------------------------------------------------