├── .dockerignore ├── doc ├── images │ ├── bgp-for-ecmp.png │ ├── ecmp-scheme.png │ ├── simple-scheme.png │ ├── bgp-for-ecmp-single.png │ └── bgp-for-ecmp-example.png ├── examples │ ├── bird-rr2.cfg │ ├── bird-core.cfg │ ├── bird-node1.cfg │ ├── bird-tor1.cfg │ ├── bird-tor2.cfg │ └── bird-rr1.cfg ├── simple-deployment-scheme.md ├── auto-allocation.md ├── deployment-examples.md ├── fail-over-optimization.md ├── operating-modes.md └── ecmp-load-balancing.md ├── helm-chart ├── externalipcontroller-0.1.0.tgz └── externalipcontroller │ ├── templates │ ├── ipclaimpool.yaml │ ├── daemonset.yaml │ ├── deployment.yaml │ ├── NOTES.txt │ └── _helpers.tpl │ ├── values.yaml │ ├── .helmignore │ └── Chart.yaml ├── .gitignore ├── Dockerfile ├── scripts ├── config.sh ├── import.sh └── push.sh ├── examples ├── ip-pool.yml ├── auth.yaml ├── nginx.yaml ├── claims │ ├── scheduler.yaml │ └── controller.yaml └── simple │ └── externalipcontroller.yaml ├── pkg ├── utils │ ├── tests_test.go │ └── tests.go ├── scheduler │ ├── node_filter.go │ ├── scheduler_test.go │ └── scheduler.go ├── workqueue │ ├── workqueue_test.go │ └── workqueue.go ├── extensions │ ├── types_test.go │ ├── register.go │ ├── testing │ │ └── client.go │ ├── types.go │ └── client.go ├── controller_test.go ├── claimcontroller │ ├── controller_test.go │ └── controller.go ├── netutils │ └── netutils.go └── controller.go ├── .travis.yml ├── cmd ├── app │ ├── root.go │ ├── naive.go │ ├── controller.go │ ├── options.go │ └── scheduler.go └── ipmanager │ └── main.go ├── test ├── e2e │ ├── basic_suite_test.go │ └── utils │ │ └── utils.go └── integration │ ├── network_suite_test.go │ └── network_test.go ├── glide.yaml ├── README.md ├── Makefile ├── LICENSE └── glide.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | test/* 2 | examples/* -------------------------------------------------------------------------------- /doc/images/bgp-for-ecmp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mirantis/k8s-externalipcontroller/HEAD/doc/images/bgp-for-ecmp.png -------------------------------------------------------------------------------- /doc/images/ecmp-scheme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mirantis/k8s-externalipcontroller/HEAD/doc/images/ecmp-scheme.png -------------------------------------------------------------------------------- /doc/images/simple-scheme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mirantis/k8s-externalipcontroller/HEAD/doc/images/simple-scheme.png -------------------------------------------------------------------------------- /doc/images/bgp-for-ecmp-single.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mirantis/k8s-externalipcontroller/HEAD/doc/images/bgp-for-ecmp-single.png -------------------------------------------------------------------------------- /doc/images/bgp-for-ecmp-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mirantis/k8s-externalipcontroller/HEAD/doc/images/bgp-for-ecmp-example.png -------------------------------------------------------------------------------- /helm-chart/externalipcontroller-0.1.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mirantis/k8s-externalipcontroller/HEAD/helm-chart/externalipcontroller-0.1.0.tgz -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/ipclaimpool.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ippool }} 2 | {{ .Values.ippool | include "ipclaimpool" }} 3 | {{ end }} 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ipcontroller 2 | ipcontroller.tar 3 | test.test 4 | vendor/ 5 | _output/ 6 | workdir/ 7 | kubeadm-dind-cluster/ 8 | 9 | *.complete 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.4 2 | MAINTAINER Dmitry Shulyak 3 | LABEL Name="k8s-externalipcontroller" Version="0.1" 4 | 5 | COPY _output/ipmanager /usr/sbin/ 6 | 7 | CMD ["sh", "-c", "/usr/sbin/ipmanager n --alsologtostderr=true --v=4 --iface=${HOST_INTERFACE}"] 8 | -------------------------------------------------------------------------------- /scripts/config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o xtrace 7 | 8 | NUM_NODES=${NUM_NODES:-2} 9 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 10 | WORKDIRECTORY=${WORKDIRECTORY:-"/tmp/kube"} 11 | mkdir -p "$WORKDIRECTORY" 12 | -------------------------------------------------------------------------------- /examples/ip-pool.yml: -------------------------------------------------------------------------------- 1 | apiVersion: ipcontroller.ext/v1 2 | kind: IpClaimPool 3 | metadata: 4 | name: test-pool 5 | spec: 6 | cidr: 192.168.0.248/29 7 | ranges: 8 | - - 192.168.0.249 9 | - 192.168.0.250 10 | - - 192.168.0.252 11 | - 192.168.0.253 12 | -------------------------------------------------------------------------------- /examples/auth.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1alpha1 3 | metadata: 4 | name: system:serviceaccounts 5 | subjects: 6 | - kind: Group 7 | name: system:serviceaccounts 8 | roleRef: 9 | kind: ClusterRole 10 | name: cluster-admin 11 | apiGroup: rbac.authorization.k8s.io 12 | -------------------------------------------------------------------------------- /pkg/utils/tests_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestEventualCondition(t *testing.T) { 11 | fakeTest := &testing.T{} 12 | assert.False(t, 13 | EventualCondition(fakeTest, time.Millisecond*100, func() bool { 14 | return 2 == 3 15 | }, "test"), 16 | ) 17 | } 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: go 3 | services: 4 | - docker 5 | env: 6 | - WORKDIRECTORY=$HOME/kube 7 | cache: 8 | directories: 9 | - $HOME/kube 10 | before_install: 11 | - mkdir -p _output/ 12 | - sudo apt-get install -y git 13 | - sudo apt-get install libpcap-dev 14 | install: 15 | - make get-deps 16 | script: make test 17 | after_success: 18 | - ./scripts/push.sh 19 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for externalipcontroller. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | replicaCount: 2 5 | image: 6 | repository: mirantis/k8s-externalipcontroller 7 | tag: latest 8 | pullPolicy: IfNotPresent 9 | scheduler: 10 | name: claimscheduler 11 | network_mask: 24 12 | controller: 13 | name: claimcontroller 14 | interface: docker0 15 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: Configure External IPs on k8s worker node(s) to provide IP connectivity. 3 | name: externalipcontroller 4 | version: 0.1.0 5 | keywords: 6 | - externalip 7 | - external-ip 8 | - externalipcontroller 9 | - external-ip-controller 10 | sources: 11 | - https://github.com/Mirantis/k8s-externalipcontroller 12 | maintainers: 13 | - name: Dmitry Shulyak 14 | email: yashulyak@gmail.com 15 | - name: Aleksei Kasatkin 16 | email: alekseyk.ru@gmail.com 17 | -------------------------------------------------------------------------------- /examples/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: nginx 5 | spec: 6 | replicas: 1 7 | template: 8 | metadata: 9 | labels: 10 | app: nginx 11 | spec: 12 | containers: 13 | - name: nginx 14 | image: nginx 15 | imagePullPolicy: IfNotPresent 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: nginxsvc 21 | labels: 22 | app: nginx 23 | spec: 24 | type: LoadBalancer 25 | ports: 26 | - port: 80 27 | protocol: TCP 28 | name: http 29 | - port: 443 30 | protocol: TCP 31 | name: https 32 | selector: 33 | app: nginx 34 | -------------------------------------------------------------------------------- /examples/claims/scheduler.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: claimscheduler 5 | spec: 6 | replicas: 1 7 | template: 8 | metadata: 9 | labels: 10 | app: claimscheduler 11 | spec: 12 | containers: 13 | - name: externalipcontroller 14 | image: mirantis/k8s-externalipcontroller 15 | imagePullPolicy: IfNotPresent 16 | command: 17 | - ipmanager 18 | - scheduler 19 | - --mask=24 20 | - --logtostderr 21 | - --v=5 22 | - --leader-elect=true 23 | - --monitor=1s 24 | - --nodefilter=fair 25 | -------------------------------------------------------------------------------- /examples/claims/controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: claimcontroller 5 | spec: 6 | template: 7 | metadata: 8 | labels: 9 | app: externalipcontroller 10 | spec: 11 | hostNetwork: true 12 | containers: 13 | - name: externalipcontroller 14 | image: mirantis/k8s-externalipcontroller 15 | imagePullPolicy: IfNotPresent 16 | securityContext: 17 | privileged: true 18 | command: 19 | - ipmanager 20 | - claimcontroller 21 | # iface is environment specific 22 | - --iface=dind0 23 | - --logtostderr 24 | - --v=5 25 | - --hb=500ms 26 | -------------------------------------------------------------------------------- /doc/examples/bird-rr2.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | debug protocols all; 3 | 4 | protocol device { 5 | scan time 2; 6 | } 7 | 8 | protocol bgp 'rack-2' { 9 | local as 65002; 10 | neighbor 10.211.2.254 port 179 as 65002; 11 | description "TOR-10.211.2.254"; 12 | multihop; 13 | rr client; 14 | import all; 15 | export all; 16 | next hop keep; 17 | source address 10.211.2.1; 18 | add paths; # For ECMP in BGP session with TOR 19 | } 20 | 21 | protocol bgp 'node-10.211.2.2' { 22 | local as 65002; 23 | neighbor 10.211.2.2 port 179 as 65002; 24 | description "node-10.211.2.2"; 25 | multihop; 26 | rr client; 27 | import all; 28 | export all; 29 | next hop keep; 30 | source address 10.211.2.1; 31 | } 32 | -------------------------------------------------------------------------------- /doc/examples/bird-core.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.254.254; 3 | 4 | debug protocols all; 5 | 6 | protocol kernel { 7 | learn; 8 | persist; 9 | scan time 2; 10 | import all; 11 | graceful restart; 12 | export all; 13 | merge paths; # For ECMP in routing table 14 | } 15 | 16 | protocol direct { 17 | interface "*"; 18 | } 19 | 20 | protocol device { 21 | scan time 2; 22 | } 23 | 24 | protocol bgp 'rack-1' { 25 | local as 65000; 26 | neighbor 192.168.192.6 as 65001; 27 | description "rack-1"; 28 | import all; 29 | export all; 30 | } 31 | 32 | protocol bgp 'rack-2' { 33 | local as 65000; 34 | neighbor 192.168.192.10 as 65002; 35 | description "rack-2"; 36 | import all; 37 | export all; 38 | } 39 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: {{ .Values.controller.name }} 5 | spec: 6 | template: 7 | metadata: 8 | labels: 9 | app: {{ .Chart.Name }} 10 | spec: 11 | hostNetwork: true 12 | containers: 13 | - name: {{ .Chart.Name }} 14 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 15 | imagePullPolicy: {{ .Values.image.pullPolicy }} 16 | securityContext: 17 | privileged: true 18 | command: 19 | - ipmanager 20 | - claimcontroller 21 | - --iface={{ .Values.controller.interface }} 22 | - --logtostderr 23 | - --v=5 24 | - --hb=500ms 25 | -------------------------------------------------------------------------------- /doc/examples/bird-node1.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.1.2 3 | debug protocols all; 4 | 5 | protocol device { 6 | scan time 2; 7 | } 8 | 9 | listen bgp address 10.211.1.2 port 179; 10 | 11 | filter exported_by_bgp { 12 | if ( ifname = "lo" ) then { 13 | if net != 127.0.0.0/8 then accept; 14 | } 15 | reject; 16 | } 17 | 18 | protocol kernel { 19 | learn; 20 | persist; 21 | scan time 2; 22 | import all; 23 | graceful restart; 24 | export all; 25 | } 26 | 27 | protocol direct { 28 | debug all; 29 | interface "-docker*", "*"; 30 | } 31 | 32 | protocol bgp 'RR-10.211.1.1' { 33 | local as 65001; 34 | neighbor 10.211.1.1 port 179 as 65001; 35 | description "RR-10.211.1.1"; 36 | import all; 37 | export filter exported_by_bgp; 38 | next hop self; 39 | source address 10.211.1.2; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /examples/simple/externalipcontroller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: externalipcontroller 5 | spec: 6 | # only single replica allowed until we will add protection against fighting for 7 | # same ip, this agent will probably become daemonset at that point 8 | replicas: 1 9 | template: 10 | metadata: 11 | labels: 12 | app: externalipcontroller 13 | spec: 14 | hostNetwork: true 15 | containers: 16 | - name: externalipcontroller 17 | image: mirantis/k8s-externalipcontroller 18 | imagePullPolicy: IfNotPresent 19 | securityContext: 20 | privileged: true 21 | env: 22 | # TODO this is specific for my vagrant environment, configmap should be used here 23 | - name: HOST_INTERFACE 24 | value: eth0 25 | -------------------------------------------------------------------------------- /doc/examples/bird-tor1.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.1.254; 3 | debug protocols all; 4 | 5 | protocol kernel { 6 | learn; 7 | persist; 8 | scan time 2; 9 | import all; 10 | graceful restart; 11 | export all; 12 | merge paths; # For ECMP in routing table 13 | } 14 | 15 | protocol direct { 16 | interface "*"; 17 | } 18 | 19 | protocol device { 20 | scan time 2; 21 | } 22 | 23 | protocol bgp 'master-192.168.192.5' { 24 | local as 65001; 25 | neighbor 192.168.192.5 as 65000; 26 | description "master"; 27 | import all; 28 | export all; 29 | next hop self; 30 | } 31 | 32 | protocol bgp 'node-01-001' { 33 | local as 65001; 34 | neighbor 10.211.1.1 as 65001; 35 | description "RR-node-01-001"; 36 | multihop; 37 | import all; 38 | export all; 39 | next hop self; 40 | add paths; # For ECMP in BGP session with RR 41 | } 42 | -------------------------------------------------------------------------------- /doc/examples/bird-tor2.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | router id 10.211.2.254; 3 | debug protocols all; 4 | 5 | protocol kernel { 6 | learn; 7 | persist; 8 | scan time 2; 9 | import all; 10 | graceful restart; 11 | export all; 12 | merge paths; # For ECMP in routing table 13 | } 14 | 15 | protocol direct { 16 | interface "*"; 17 | } 18 | 19 | protocol device { 20 | scan time 2; 21 | } 22 | 23 | protocol bgp 'master-192.168.192.9' { 24 | local as 65002; 25 | neighbor 192.168.192.9 as 65000; 26 | description "master"; 27 | import all; 28 | export all; 29 | next hop self; 30 | } 31 | 32 | protocol bgp 'node-02-001' { 33 | local as 65002; 34 | neighbor 10.211.2.1 as 65002; 35 | description "RR-node-02-001"; 36 | multihop; 37 | import all; 38 | export all; 39 | next hop self; 40 | add paths; # For ECMP in BGP session with RR 41 | } 42 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.scheduler.name }} 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | app: {{ .Values.scheduler.name }} 13 | spec: 14 | containers: 15 | - name: {{ .Chart.Name }} 16 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 17 | imagePullPolicy: {{ .Values.image.pullPolicy }} 18 | command: 19 | - ipmanager 20 | - scheduler 21 | - --mask={{ .Values.scheduler.network_mask }} 22 | - --logtostderr 23 | - --v=5 24 | - --leader-elect=true 25 | - --monitor=4s 26 | - --nodefilter=fair 27 | -------------------------------------------------------------------------------- /cmd/app/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import "github.com/spf13/cobra" 18 | 19 | var Root = &cobra.Command{ 20 | Use: "ipmanager", 21 | Short: "Application to manage IPs assignment to k8s services", 22 | Run: func(cmd *cobra.Command, args []string) {}, 23 | } 24 | -------------------------------------------------------------------------------- /test/e2e/basic_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package e2e 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestBasicFeatures(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Basic") 27 | } 28 | -------------------------------------------------------------------------------- /test/integration/network_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestNetwork(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Network") 27 | } 28 | -------------------------------------------------------------------------------- /scripts/import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o xtrace 7 | 8 | CURDIR=$(dirname "${BASH_SOURCE[0]}") 9 | source "${CURDIR}/config.sh" 10 | 11 | IMAGE_REPO=${IMAGE_REPO:-mirantis/k8s-externalipcontroller} 12 | IMAGE_TAG=${IMAGE_TAG:-latest} 13 | 14 | function import-image { 15 | echo "Export docker image and import it on a dind node dind_node_1" 16 | CONTAINERID="$(docker create "${IMAGE_REPO}":"${IMAGE_TAG}" bash)" 17 | mkdir -p _output/ 18 | docker export "${CONTAINERID}" > _output/ipcontroller.tar 19 | # TODO implement it as a provider (e.g source functions) 20 | 21 | for i in $(seq 1 "${NUM_NODES}"); do 22 | docker cp _output/ipcontroller.tar dind_node_"$i":/tmp 23 | docker exec -ti dind_node_"$i" docker import /tmp/ipcontroller.tar "${IMAGE_REPO}":"${IMAGE_TAG}" 24 | done 25 | echo "Finished copying docker image to dind nodes" 26 | } 27 | 28 | import-image 29 | -------------------------------------------------------------------------------- /doc/examples/bird-rr1.cfg: -------------------------------------------------------------------------------- 1 | log stderr all; 2 | debug protocols all; 3 | 4 | protocol device { 5 | scan time 2; 6 | } 7 | 8 | protocol bgp 'rack-1' { 9 | local as 65001; 10 | neighbor 10.211.1.254 port 179 as 65001; 11 | description "TOR-10.211.1.254"; 12 | multihop; 13 | rr client; 14 | import all; 15 | export all; 16 | next hop keep; 17 | source address 10.211.1.1; 18 | add paths; # For ECMP in BGP session with TOR 19 | } 20 | 21 | protocol bgp 'node-10.211.1.2' { 22 | local as 65001; 23 | neighbor 10.211.1.2 port 179 as 65001; 24 | description "node-10.211.1.2"; 25 | multihop; 26 | rr client; 27 | import all; 28 | export all; 29 | next hop keep; 30 | source address 10.211.1.1; 31 | } 32 | 33 | protocol bgp 'node-10.211.1.3' { 34 | local as 65001; 35 | neighbor 10.211.1.3 port 179 as 65001; 36 | description "node-10.211.1.3"; 37 | multihop; 38 | rr client; 39 | import all; 40 | export all; 41 | next hop keep; 42 | source address 10.211.1.1; 43 | } 44 | -------------------------------------------------------------------------------- /scripts/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-} 8 | 9 | function push-to-docker { 10 | if [ "$TRAVIS_PULL_REQUEST_BRANCH" != "" ]; then 11 | echo "Processing PR $TRAVIS_PULL_REQUEST_BRANCH" 12 | exit 0 13 | fi 14 | 15 | docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" 16 | 17 | set -o xtrace 18 | TRAVIS_BRANCH=${TRAVIS_BRANCH:-} 19 | local branch=$TRAVIS_BRANCH 20 | echo "Using git branch $branch" 21 | 22 | if [ "$branch" == "master" ]; then 23 | echo "Pushing with tag - latest" 24 | docker push mirantis/k8s-externalipcontroller:latest; 25 | fi 26 | 27 | if [ "${branch:0:8}" == "release-" ]; then 28 | echo "Pushing from release branch with tag - $branch" 29 | docker tag mirantis/k8s-externalipcontroller mirantis/k8s-externalipcontroller:"$branch"; 30 | docker push mirantis/k8s-externalipcontroller:"$branch"; 31 | fi 32 | } 33 | 34 | push-to-docker 35 | -------------------------------------------------------------------------------- /cmd/ipmanager/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "log" 19 | 20 | "github.com/Mirantis/k8s-externalipcontroller/cmd/app" 21 | 22 | "k8s.io/kubernetes/pkg/util/flag" 23 | ) 24 | 25 | func main() { 26 | flag.InitFlags() 27 | if err := app.AppOpts.CheckFlags(); err != nil { 28 | log.Fatal(err) 29 | } 30 | if err := app.Root.Execute(); err != nil { 31 | log.Fatal(err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | For general information about External IP Controller application, see: 2 | https://github.com/Mirantis/k8s-externalipcontroller 3 | 4 | For more details about using External IP Controller application, see: 5 | https://github.com/Mirantis/k8s-externalipcontroller/blob/master/doc 6 | 7 | This chart allows running External IP Controller application in claims mode. 8 | For more details about application operational modes, see: 9 | https://github.com/Mirantis/k8s-externalipcontroller/blob/master/doc/operating-modes.md 10 | 11 | In order to auto-create IpClaimPool one needs to add ippool.cidr and ippool.ranges into Values 12 | where ippool.ranges is optional. 13 | definition example: 14 | 15 | ippool: 16 | cidr: 192.168.0.248/29 17 | ranges: 18 | - - 192.168.0.249 19 | - 192.168.0.250 20 | - - 192.168.0.252 21 | - 192.168.0.253 22 | 23 | For more details about auto allocation of external IPs, see: 24 | https://github.com/Mirantis/k8s-externalipcontroller/blob/master/doc/auto-allocation.md 25 | -------------------------------------------------------------------------------- /pkg/utils/tests.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/stretchr/testify/assert" 21 | ) 22 | 23 | func EventualCondition(t assert.TestingT, wait time.Duration, comp assert.Comparison, msgAndArgs ...interface{}) bool { 24 | after := time.After(wait) 25 | for { 26 | select { 27 | case <-after: 28 | return assert.Condition(t, comp, msgAndArgs...) 29 | case <-time.Tick(10 * time.Millisecond): 30 | if comp() { 31 | return true 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /helm-chart/externalipcontroller/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- $name := default .Chart.Name .Values.nameOverride -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 16 | {{- end -}} 17 | 18 | {{/* 19 | Create IpClaimPool resource to provide IP pool for the IP claim allocator. 20 | */}} 21 | {{- define "ipclaimpool" -}} 22 | apiVersion: ipcontroller.ext/v1 23 | kind: IpClaimPool 24 | metadata: 25 | name: default-pool 26 | spec: 27 | cidr: {{ .cidr }} 28 | {{ if .ranges }} 29 | {{ . | include "ipranges" }} 30 | {{- end -}} 31 | {{- end -}} 32 | 33 | {{/* 34 | Define IP ranges list for IpClaimPool resource. 35 | */}} 36 | {{- define "ipranges" -}} 37 | ranges: 38 | {{- range .ranges }} 39 | - - {{ index . 0 | quote }} 40 | - {{ index . 1 | quote }} 41 | {{- end }} 42 | {{- end -}} 43 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/Mirantis/k8s-externalipcontroller 2 | import: 3 | - package: github.com/golang/glog 4 | - package: github.com/vishvananda/netlink 5 | - package: k8s.io/apimachinery 6 | - package: k8s.io/client-go 7 | version: v4.0.0 8 | subpackages: 9 | - kubernetes 10 | - pkg/api 11 | - pkg/runtime 12 | - pkg/watch 13 | - rest 14 | - tools/cache 15 | - package: github.com/onsi/ginkgo 16 | - package: github.com/onsi/gomega 17 | version: v1.0 18 | - package: github.com/coreos/etcd 19 | version: v3.1.0-rc.0 20 | subpackages: 21 | - client 22 | - package: github.com/stretchr/testify 23 | version: v1.1.4 24 | - package: github.com/pmezard/go-difflib 25 | version: v1.0.0 26 | subpackages: 27 | - difflib 28 | - package: k8s.io/kubernetes 29 | version: v1.6.0-alpha.0 30 | subpackages: 31 | - pkg/client/leaderelection 32 | - pkg/apis/componentconfig 33 | - pkg/client/unversioned/remotecommand 34 | - pkg/util/flag 35 | - pkg/client/record 36 | - pkg/client/restclient 37 | - pkg/client/clientset_generated/internalclientset 38 | - pkg/client/leaderelection/resourcelock 39 | - package: github.com/spf13/cobra 40 | - package: github.com/google/gopacket 41 | version: v1.1.12 42 | - package: github.com/emicklei/go-restful 43 | version: v1.2 44 | - package: k8s.io/apiextensions-apiserver 45 | - package: k8s.io/api 46 | -------------------------------------------------------------------------------- /pkg/scheduler/node_filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scheduler 16 | 17 | import ( 18 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 19 | 20 | "github.com/golang/glog" 21 | ) 22 | 23 | func (s *ipClaimScheduler) getFairNode(ipnodes []*extensions.IpNode) *extensions.IpNode { 24 | counter := make(map[string]int) 25 | for _, key := range s.claimStore.ListKeys() { 26 | obj, _, err := s.claimStore.GetByKey(key) 27 | claim := obj.(*extensions.IpClaim) 28 | if err != nil { 29 | glog.Errorln(err) 30 | continue 31 | } 32 | if claim.Spec.NodeName == "" { 33 | continue 34 | } 35 | counter[claim.Spec.NodeName]++ 36 | } 37 | var min *extensions.IpNode 38 | minCount := -1 39 | for _, node := range ipnodes { 40 | count := counter[node.Metadata.Name] 41 | if minCount == -1 || count < minCount { 42 | minCount = count 43 | min = node 44 | } 45 | } 46 | return min 47 | } 48 | 49 | func (s *ipClaimScheduler) getFirstAliveNode(ipnodes []*extensions.IpNode) *extensions.IpNode { 50 | return ipnodes[0] 51 | } 52 | -------------------------------------------------------------------------------- /pkg/workqueue/workqueue_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package workqueue 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestQueue(t *testing.T) { 23 | queue := NewQueue() 24 | queue.Add(1) 25 | queue.Add(2) 26 | queue.Remove(1) 27 | item, _ := queue.Get() 28 | if citem := item.(int); citem != 2 { 29 | t.Errorf("item expected to be 2 - %v", citem) 30 | } 31 | queue.Close() 32 | item, closed := queue.Get() 33 | if item != nil { 34 | t.Errorf("expected to return nil if empty") 35 | } 36 | if !closed { 37 | t.Errorf("queue expected to be closed") 38 | } 39 | } 40 | 41 | func TestProcessQueue(t *testing.T) { 42 | queue := ProcessingQueue{NewQueue()} 43 | queue.Add(1) 44 | queue.Add(2) 45 | queue.Process(func(item interface{}) error { 46 | if citem := item.(int); citem != 1 { 47 | t.Errorf("item expected to be 1 - %v - %v", citem, queue.queue) 48 | } 49 | return fmt.Errorf("test") 50 | }) 51 | queue.Process(func(item interface{}) error { 52 | if citem := item.(int); citem != 2 { 53 | t.Errorf("item expected to be 2 - %v - %v", citem, queue.queue) 54 | } 55 | return nil 56 | }) 57 | queue.Process(func(item interface{}) error { 58 | if citem := item.(int); citem != 1 { 59 | t.Errorf("item expected to be 1 - %v - %v", citem, queue.queue) 60 | } 61 | return nil 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /cmd/app/naive.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "os" 19 | 20 | "github.com/golang/glog" 21 | "github.com/spf13/cobra" 22 | "k8s.io/client-go/rest" 23 | "k8s.io/client-go/tools/clientcmd" 24 | 25 | externalip "github.com/Mirantis/k8s-externalipcontroller/pkg" 26 | ) 27 | 28 | func init() { 29 | Root.AddCommand(Naive) 30 | } 31 | 32 | var Naive = &cobra.Command{ 33 | Aliases: []string{"n"}, 34 | Use: "naivecontroller", 35 | RunE: func(cmd *cobra.Command, args []string) error { 36 | return InitNaiveController() 37 | }, 38 | } 39 | 40 | func InitNaiveController() error { 41 | kubeconfig := AppOpts.Kubeconfig 42 | iface := AppOpts.Iface 43 | mask := AppOpts.Mask 44 | 45 | glog.V(4).Infof("Starting external ip controller using link: %s and mask: /%s", iface, mask) 46 | stopCh := make(chan struct{}) 47 | 48 | var err error 49 | var config *rest.Config 50 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 51 | if err != nil { 52 | glog.Errorf("Error parsing config. %v", err) 53 | return err 54 | } 55 | 56 | host, err := os.Hostname() 57 | if err != nil { 58 | return err 59 | } 60 | 61 | c, err := externalip.NewExternalIpController(config, host, iface, mask, AppOpts.ResyncInterval) 62 | if err != nil { 63 | return err 64 | } 65 | c.Run(stopCh) 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /doc/simple-deployment-scheme.md: -------------------------------------------------------------------------------- 1 | Simple Deployment Scheme 2 | ======================== 3 | 4 | ## Introduction 5 | 6 | One of the possible ways to expose k8s services on a bare metal deployments is 7 | using External IPs. Each node runs a kube-proxy process which programs `iptables` 8 | rules to trap access to External IPs and redirect them to the correct backends. 9 | 10 | So in order to access k8s service from the outside we just need to route public 11 | traffic to one of k8s worker nodes which has `kube-proxy` running and thus has 12 | needed `iptables` rules for External IPs. 13 | 14 | ## Proposed solution 15 | 16 | External IP controller is k8s application which is deployed on top of k8s 17 | cluster and which configures External IPs on k8s worker node(s) to provide 18 | IP connectivity. 19 | 20 | This document describes a simple deployment scheme for External IP controller. 21 | 22 | We assume that: 23 | * Public network (network for External IPs) is routable to nodes. 24 | * We want to expose services to external clients via external IPs. 25 | 26 | ## Deployment scheme 27 | 28 | ![Deployment scheme](images/simple-scheme.png) 29 | 30 | Description: 31 | * External IP controller kubernetes application is running on one of the nodes 32 | (`replicas=1`). 33 | * On start it pulls information about services from `kube-api` and brings up 34 | all External IPs on the specified interface (eth0 in our example above). 35 | * It watches `kube-api` for updates in services with External IPs and: 36 | * When new External IPs appear it brings them up. 37 | * When service is removed it removes appropriate External IPs from the 38 | interface. 39 | * Kubernetes provides fail-over for External IP controller. Since we have 40 | `replicas` set to 1, then we'll have only one instance running in a cluster to 41 | avoid IPs duplication. And when there's a problem with k8s node, External IP 42 | controller will be spawned on a new k8s worker node and bring External IPs up 43 | on that node. 44 | -------------------------------------------------------------------------------- /doc/auto-allocation.md: -------------------------------------------------------------------------------- 1 | Auto allocation of external IPs 2 | =============================== 3 | 4 | Now users can utilize auto allocation feature. 5 | The feature is only available in Claims mode (see ``Basic Modules and Operating 6 | Modes`` document for more detail). 7 | 8 | In order to do that, users must first provide 9 | IP pool for the allocator by creation of IpClaimPool resources. 10 | Here is example description for one in yaml format: 11 | 12 | ```yaml 13 | apiVersion: ipcontroller.ext/v1 14 | kind: IpClaimPool 15 | metadata: 16 | name: test-pool 17 | spec: 18 | cidr: 192.168.0.248/29 19 | ranges: 20 | - - 192.168.0.249 21 | - 192.168.0.250 22 | - - 192.168.0.252 23 | - 192.168.0.253 24 | ``` 25 | 26 | Few words about the Spec: CIDR is mandatory to supply. 27 | Ranges can be omitted (range of the whole network defined in "CIDR" field is used then). 28 | Exclusion of particular addresses is done via specifying multiple ranges. 29 | In the above example, address 192.168.0.251 is not processed by the allocator. 30 | 31 | In order to enable auto allocation for particular services, users 32 | must annotate them with following key-value pair - "external-ip = auto" 33 | either on creation or while the service is running (via `kubectl annotate`). 34 | After that, exactly one external IP address will be allocated for the service 35 | and IP claim will be created. Allocated address is then stored in 'allocated' 36 | field of the IP pool object. 37 | 38 | Users can create several IP pools. Allocator will process all of them in the 39 | same manner (looking for the first available IP) and with no regard for pools 40 | order, that is users cannot control which pool an address is fetched from. 41 | 42 | When service is deleted the allocation is freed and returned to the pool of 43 | usable addresses again. Corresponding IP claim is removed. 44 | 45 | Here is a brief demo of the functionality. 46 | 47 | [![asciicast](https://asciinema.org/a/6uyrkfn66nufzpuhrwt1veshb.png)](https://asciinema.org/a/6uyrkfn66nufzpuhrwt1veshb) 48 | -------------------------------------------------------------------------------- /cmd/app/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "os" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/claimcontroller" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 23 | 24 | "github.com/spf13/cobra" 25 | "k8s.io/client-go/rest" 26 | "k8s.io/client-go/tools/clientcmd" 27 | ) 28 | 29 | func init() { 30 | Root.AddCommand(Controller) 31 | } 32 | 33 | var Controller = &cobra.Command{ 34 | Aliases: []string{"c"}, 35 | Use: "claimcontroller", 36 | RunE: func(cmd *cobra.Command, args []string) error { 37 | return InitController() 38 | }, 39 | } 40 | 41 | func InitController() error { 42 | var err error 43 | var config *rest.Config 44 | kubeconfig := AppOpts.Kubeconfig 45 | iface := AppOpts.Iface 46 | hostname := AppOpts.Hostname 47 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 48 | if err != nil { 49 | return err 50 | } 51 | uid := hostname 52 | if hostname == "" { 53 | uid, err = os.Hostname() 54 | } 55 | if err != nil { 56 | return err 57 | } 58 | stop := make(chan struct{}) 59 | c, err := claimcontroller.NewClaimController(iface, uid, config, AppOpts.ResyncInterval, AppOpts.HeartbeatInterval) 60 | if err != nil { 61 | return err 62 | } 63 | err = extensions.EnsureCRDsExist(config) 64 | if err != nil { 65 | return err 66 | } 67 | err = extensions.WaitCRDsEstablished(config, 10*time.Second) 68 | if err != nil { 69 | return err 70 | } 71 | c.Run(stop) 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /cmd/app/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package app 15 | 16 | import ( 17 | "fmt" 18 | "errors" 19 | "strings" 20 | "time" 21 | 22 | "k8s.io/kubernetes/pkg/apis/componentconfig" 23 | "k8s.io/kubernetes/pkg/client/leaderelection" 24 | 25 | "github.com/spf13/pflag" 26 | ) 27 | 28 | type options struct { 29 | Hostname string 30 | Iface string 31 | Kubeconfig string 32 | Mask string 33 | NodeFilter string 34 | 35 | HeartbeatInterval time.Duration 36 | MonitorInterval time.Duration 37 | ResyncInterval time.Duration 38 | 39 | LeaderElection componentconfig.LeaderElectionConfiguration 40 | } 41 | 42 | var AppOpts = options{} 43 | 44 | var NodeFilters = []string{ 45 | "fair", 46 | "first-alive", 47 | } 48 | 49 | func init() { 50 | AppOpts.AddFlags(pflag.CommandLine) 51 | } 52 | 53 | func (o *options) AddFlags(fs *pflag.FlagSet) { 54 | fs.StringVar(&o.Iface, "iface", "eth0", "Current interface will be used to assign ip addresses") 55 | fs.StringVar(&o.Mask, "mask", "32", "mask part of the cidr") 56 | fs.StringVar(&o.Kubeconfig, "kubeconfig", "", "kubeconfig to use with kubernetes client") 57 | fs.StringVar(&o.Hostname, "hostname", "", "We will use os.Hostname if none provided") 58 | filterList := strings.Join(NodeFilters, "|") 59 | fs.StringVar(&o.NodeFilter, "nodefilter", NodeFilters[0], fmt.Sprintf("Possible values: %s. We will use '%s' if none was provided.", filterList, NodeFilters[0])) 60 | fs.DurationVar(&o.ResyncInterval, "resync", 20*time.Second, "Time to resync state for all ips") 61 | fs.DurationVar(&o.HeartbeatInterval, "hb", 2*time.Second, "How often to send heartbeats from controllers?") 62 | fs.DurationVar(&o.MonitorInterval, "monitor", 4*time.Second, "How often to check controllers liveness?") 63 | o.LeaderElection = leaderelection.DefaultLeaderElectionConfiguration() 64 | leaderelection.BindFlags(&o.LeaderElection, fs) 65 | } 66 | 67 | func (o *options) CheckFlags() error { 68 | for _, f := range NodeFilters { 69 | if o.NodeFilter == f { 70 | return nil 71 | } 72 | } 73 | return errors.New("Incorrect node filter is provided") 74 | } -------------------------------------------------------------------------------- /doc/deployment-examples.md: -------------------------------------------------------------------------------- 1 | Application Deployment Examples 2 | =============================== 3 | 4 | The application can be run in one of the following operating modes: 5 | * Simple mode 6 | * Claims mode 7 | 8 | You can get more information about operating modes in the ``Basic Modules and 9 | Operating Modes`` document. 10 | 11 | Let's look at how to deploy the application in a k8s cluster. 12 | 13 | We assume that: 14 | * Public network (network for External IPs) is routable to nodes. 15 | * We want to expose services to external clients via external IPs. 16 | 17 | There is a set of deployment configuration examples in the `examples` directory 18 | that we will use here. 19 | 20 | The following is the description of CLI command flow that is used to deploy 21 | and operate the application. However, it is the same flow that is in use for 22 | any other k8s application. 23 | 24 | In order to run the application, one needs to use `kubectl apply` providing 25 | the appropriate configuration file(s): 26 | * Simple mode: `kubectl apply -f examples/simple/externalipcontroller.yaml` 27 | * Claims mode: `kubectl apply -f examples/claims/controller.yaml -f examples/claims/scheduler.yaml` 28 | You can get more information regarding deployment of the application in simple 29 | mode from the ``Simple Deployment Scheme`` document. 30 | 31 | To check running pods, run `kubectl get pods -o wide`. Application pods should 32 | appear in the output:: 33 | 34 | NAME READY STATUS RESTARTS AGE IP NODE 35 | claimcontroller-5k4lx 1/1 Running 0 54m 10.250.1.13 k8s-03 36 | claimcontroller-95xhc 1/1 Running 0 54m 10.250.1.12 k8s-02 37 | claimcontroller-l5rhr 1/1 Running 0 54m 10.250.1.11 k8s-01 38 | claimscheduler-1170328893-5m2nb 1/1 Running 0 54m 10.233.75.195 k8s-05 39 | claimscheduler-1170328893-pwmhd 1/1 Running 0 54m 10.233.126.2 k8s-03 40 | 41 | To check third party resources (TPR), that the application uses, run 42 | `kubectl get thirdpartyresources`. If everything went OK, the following should 43 | be in the list:: 44 | 45 | NAME DESCRIPTION VERSION(S) 46 | ip-claim-pool.ipcontroller.ext v1 47 | ip-claim.ipcontroller.ext v1 48 | ip-node.ipcontroller.ext v1 49 | 50 | Each of these resources can be got/altered separately. 51 | E.g., `kubectl get ipnode` should return all the nodes where the controller 52 | modules are running. 53 | 54 | There is an Nginx deployment example in `examples/nginx.yaml` that can be used 55 | to test External IPs functioning. One can alter `externalIPs` section according 56 | to environment settings and run Nginx server using `kubectl apply -f` command. 57 | Given set of IPs should be set up by the External IP Controller then. It can be 58 | checked with `curl` and by retrieving `ipclaim` TPR:: 59 | 60 | $ kubectl get ipclaim --show-labels 61 | NAME KIND LABELS 62 | 10-0-0-7-24 IpClaim.v1.ipcontroller.ext ipnode=k8s-01 63 | 10-0-0-8-24 IpClaim.v1.ipcontroller.ext ipnode=k8s-02 64 | 65 | For the information regarding `ipclaimpool` resource, please refer to the ``Auto 66 | allocation of external IPs`` document. 67 | 68 | To remove the application, please delete corresponding daemon set and deployment 69 | using `kubectl delete` command. 70 | -------------------------------------------------------------------------------- /pkg/workqueue/workqueue.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package workqueue 16 | 17 | import ( 18 | "sync" 19 | ) 20 | 21 | type QueueAddType interface { 22 | Add(interface{}) 23 | } 24 | 25 | type QueueType interface { 26 | QueueAddType 27 | Get() (interface{}, bool) 28 | Done(interface{}) 29 | Remove(interface{}) 30 | Close() 31 | Len() int 32 | } 33 | 34 | func NewQueue() *Queue { 35 | return &Queue{ 36 | cond: sync.NewCond(&sync.Mutex{}), 37 | added: map[interface{}]bool{}, 38 | processing: map[interface{}]bool{}, 39 | queue: []interface{}{}, 40 | } 41 | } 42 | 43 | type Queue struct { 44 | cond *sync.Cond 45 | added map[interface{}]bool 46 | processing map[interface{}]bool 47 | closed bool 48 | queue []interface{} 49 | } 50 | 51 | func (n *Queue) Add(item interface{}) { 52 | n.cond.L.Lock() 53 | defer n.cond.L.Unlock() 54 | if n.closed { 55 | return 56 | } 57 | if _, exists := n.added[item]; exists { 58 | return 59 | } 60 | n.added[item] = true 61 | if _, exists := n.processing[item]; exists { 62 | return 63 | } 64 | n.queue = append(n.queue, item) 65 | n.cond.Signal() 66 | } 67 | 68 | func (n *Queue) Len() int { 69 | return len(n.queue) 70 | } 71 | 72 | func (n *Queue) Close() { 73 | n.cond.L.Lock() 74 | defer n.cond.L.Unlock() 75 | n.closed = true 76 | n.cond.Broadcast() 77 | } 78 | 79 | func (n *Queue) Get() (item interface{}, quit bool) { 80 | n.cond.L.Lock() 81 | defer n.cond.L.Unlock() 82 | 83 | if len(n.queue) == 0 && !n.closed { 84 | n.cond.Wait() 85 | } 86 | 87 | if len(n.queue) == 0 { 88 | return nil, n.closed 89 | } 90 | 91 | for { 92 | item, n.queue = n.queue[0], n.queue[1:] 93 | // item was removed and shouldn't be processed 94 | if _, exists := n.added[item]; !exists { 95 | if len(n.queue) == 0 { 96 | return nil, n.closed 97 | } 98 | continue 99 | } 100 | break 101 | } 102 | n.processing[item] = true 103 | delete(n.added, item) 104 | return item, false 105 | } 106 | 107 | func (n *Queue) Done(item interface{}) { 108 | n.cond.L.Lock() 109 | defer n.cond.L.Unlock() 110 | 111 | delete(n.processing, item) 112 | 113 | if _, exists := n.added[item]; exists { 114 | n.queue = append(n.queue, item) 115 | n.cond.Signal() 116 | } 117 | } 118 | 119 | // Remove will prevent item from being processed 120 | func (n *Queue) Remove(item interface{}) { 121 | n.cond.L.Lock() 122 | defer n.cond.L.Unlock() 123 | 124 | if _, exists := n.added[item]; exists { 125 | delete(n.added, item) 126 | } 127 | } 128 | 129 | type ProcessType interface { 130 | Process(func(item interface{}) error) 131 | } 132 | 133 | type ProcessingQueue struct { 134 | *Queue 135 | } 136 | 137 | func (p *ProcessingQueue) Process(f func(item interface{}) error) error { 138 | item, _ := p.Get() 139 | defer p.Done(item) 140 | if err := f(item); err != nil { 141 | p.Add(item) 142 | return err 143 | } 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | External IP Controller [![Build Status](https://travis-ci.org/Mirantis/k8s-externalipcontroller.svg?branch=master)](https://travis-ci.org/Mirantis/k8s-externalipcontroller) 2 | ====================== 3 | 4 | ## Introduction 5 | 6 | One of the possible ways to expose k8s services on a bare metal deployments is 7 | using External IPs. Each node runs a kube-proxy process which programs `iptables` 8 | rules to trap requests to External IPs and redirect them to the correct backends. 9 | 10 | So, in order to access k8s service from the outside, we just need to route public 11 | traffic to one of the k8s worker nodes which have `kube-proxy` running and thus 12 | have all the needed `iptables` rules for External IPs configured. 13 | 14 | ## Proposed solution 15 | 16 | External IP Controller is a k8s application which is deployed on top of k8s 17 | cluster and which configures External IPs on k8s worker node(s) to provide 18 | IP connectivity. 19 | 20 | ## Demo 21 | 22 | [![asciicast](https://asciinema.org/a/95449.png)](https://asciinema.org/a/95449) 23 | 24 | How to run tests 25 | ================ 26 | 27 | Install dependencies and prepare kubernetes dind cluster. It is supposed that 28 | Go v.1.7.x has been installed already. 29 | ``` 30 | make get-deps 31 | ``` 32 | 33 | Build necessary images and run tests. 34 | ``` 35 | make test 36 | ``` 37 | 38 | Use ```make help``` to see all the options available. 39 | 40 | How to start using this? 41 | ======================== 42 | 43 | Both controller and scheduller operate on third party resources and require them 44 | to be created. Since kubernetes 1.7 most of the installations enable RBAC. 45 | For this reason we need to grant our application correct permissions. For 46 | testing envrionment you can use: 47 | ``` 48 | kubectl apply -f examples/auth.yaml 49 | ``` 50 | 51 | In case you are using kubeadm dind environment - deploy claim controller and scheduller like this: 52 | ``` 53 | kubectl apply -f examples/claims/ 54 | ``` 55 | For any other environment you need to ensure that `--iface` option in 56 | examples/claims/controller.yaml file is correct. This interface will be used for IP assignment. 57 | 58 | If you want to use auto allocation from IP pool - you need to create atleast one such pool. 59 | We provided an example in file `examples/ip-pool.yml`. It can be applied with kubectl after 60 | third party resources will be created. 61 | We are not resyncing services after pool was created, so please ensure that it is created 62 | before you will start requesting IPs. 63 | 64 | We also have one basic example with nginx service and pods - `examples/nginx.yaml`. This example 65 | creates deployment for nginx with single replica and service of type LoadBalancer. 66 | 67 | For each service that require ip we will create ipclaim object. You can list all ipclaims with: 68 | ``` 69 | kubectl get ipclaims 70 | ``` 71 | 72 | Notes on CI and end-to-end tests 73 | ================================ 74 | In tests we want to verify that IPs are reachable remotely. For this purpose we are using --testlink option in e2e tests. 75 | During the tests we will configure that link with IP from a network that is used in tests. 76 | This is also the reason why we are running e2e tests with sudo. 77 | The requirement here is that all kubernetes nodes must be in the same L2 domain. 78 | In our application we are assigning IPs to a node. In dind-based setup those nodes are regular containers. 79 | Therefore to guarantee connectivity in our CI we need to assign IP on a bridge used by docker. 80 | 81 | For simplicity we want to limit number of running ipcontrollers to 2. To make it work with kubeadm-dind-cluster 82 | we have to set label ipcontroller= on kube workers. And in the test we are using this label as node selector for daemonset pods. 83 | -------------------------------------------------------------------------------- /doc/fail-over-optimization.md: -------------------------------------------------------------------------------- 1 | Fail-Over Optimization 2 | ====================== 3 | 4 | External IP fail-over functionality depends on both External IP Controller 5 | application capabilities and a number of k8s parameters. 6 | Application itself provides fail-over capabilities in claims mode. 7 | There is a set of k8s parameters that affect pods respawning lag after some 8 | node goes offline. These parameters have different effect on application 9 | fail-over functionality depending on application operating mode. 10 | 11 | ## Pods Respawning and Corresponding k8s Parameters 12 | 13 | Kubernetes cares only about pods availability including pods where External IP 14 | Controller application is running. So, when some node goes offline k8s detects 15 | that and respawns the corresponding pods on some another node. 16 | 17 | The following parameters affect pods respawning lag: 18 | * `--node-status-update-frequency` - kubelet's parameter, 10s by default; 19 | * `--node-monitor-period` - parameter of kube-controller-manager, 5s by default; 20 | * `--node-monitor-grace-period` - parameter of kube-controller-manager, 40s by 21 | default; 22 | * `--pod-eviction-timeout` - parameter of kube-controller-manager, 5m by default; 23 | Please refer to kubernetes docs for more detail. In short, `node-monitor-grace-period` 24 | determines how fast node will be considered as being offline, 25 | `node-status-update-frequency` and `node-monitor-period` should be proportionally 26 | smaller than that. `pod-eviction-timeout` determines when pods will be 27 | respawned on some other node after failed node status will be changed to `NotReady`. 28 | Changing default values to `--node-status-update-frequency=4s`, 29 | `--node-monitor-period=2s`, `--node-monitor-grace-period=16s`, 30 | `--pod-eviction-timeout=20s` allows to decrease pods respawning lag from 5 minutes 31 | to about 20 seconds. These values were tested on a lab with 5-6 nodes clusters. 32 | 33 | ## Simple Mode 34 | 35 | In simple mode, kubernetes will respawn IP Controller pods so that application 36 | is able to continue operating properly (will respawn corresponding IPs on a new 37 | node). 38 | As there is the only controller part which is running in simple mode, so it is 39 | just kubernetes who cares about controllers' availability. 40 | 41 | ## Claims Mode 42 | 43 | In claims mode, kubernetes will respawn IP Scheduler pods and IP Controller 44 | pods. But in claims mode it is not so critical for the application as leader 45 | election support for schedulers and monitoring of controllers reachability 46 | improves availability of the application. 47 | Thanks to the leader election, one scheduler will be active at any 48 | particular time. If the active scheduler becomes unavailable, then 49 | another instance of scheduler becomes active. So, scheduler remains functional 50 | while at least one operational node has IP Scheduler pods. Scheduler module 51 | monitors controllers availability and reschedules IP claims to healthy 52 | controllers from unavailable ones. 53 | In overall, application will continue to work properly while at least one 54 | scheduler and one controller remain operational. 55 | There is a set of application parameters that affect IP fail-over: 56 | `hb`, `leader-elect` and `monitor`. For parameters description, please refer to 57 | "Basic Modules and Operating Modes" document for more detail. 58 | 59 | ## Known Issues 60 | 61 | The following issues with kubernetes were discovered when any of master or etcd 62 | nodes goes down. 63 | Regardless of the settings described above there are chances that pods from 64 | failed nodes will be respawned with about 15-20 minutes lag. 65 | Another issue is that kube-proxy does not resetup routes so applications cannot 66 | access kube-API when some of the master nodes go down. 67 | This behaviour was observed with k8s v.1.5.x and calico v1.1.0-rc3, v1.1.0-rc5 68 | within 25-30% cases. 69 | -------------------------------------------------------------------------------- /pkg/extensions/types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func checkFreeIP(t *testing.T, p *IpClaimPool, expected string) { 22 | actual, err := p.AvailableIP() 23 | if err != nil { 24 | t.Errorf("Error must not occur during AvailableIP() method; details --> %v\n", err) 25 | } 26 | 27 | if expected != actual { 28 | t.Errorf("Actual free IP %v is not as expected %v\n", actual, expected) 29 | } 30 | } 31 | 32 | func checkNoFreeIPError(t *testing.T, p *IpClaimPool) { 33 | ip, err := p.AvailableIP() 34 | 35 | if len(ip) != 0 { 36 | t.Error("AvailableIP must return empty string as IP value in case there is no free IP") 37 | } 38 | if err == nil { 39 | t.Error("AvailableIP must return error in case there is no free IP") 40 | } 41 | expectedErrMsg := "There is no free IP left in the pool" 42 | if err.Error() != expectedErrMsg { 43 | t.Errorf("Message of the returned by AvailableIP error is not as expected ('%v'). Actual: %v", expectedErrMsg, err) 44 | } 45 | } 46 | 47 | func TestIpClaimPoolAvailableIPFromCIDR(t *testing.T) { 48 | cidr := "192.168.16.248/29" 49 | allocated := map[string]string{ 50 | "192.168.16.249": "test-claim-249", 51 | "192.168.16.250": "test-claim-250", 52 | "192.168.16.252": "test-claim-252", 53 | "192.168.16.253": "test-claim-253", 54 | } 55 | expectedIP := "192.168.16.251" 56 | 57 | ClaimPool := &IpClaimPool{ 58 | Spec: IpClaimPoolSpec{ 59 | CIDR: cidr, 60 | Ranges: nil, 61 | Allocated: allocated, 62 | }, 63 | } 64 | 65 | checkFreeIP(t, ClaimPool, expectedIP) 66 | 67 | allocated["192.168.16.254"] = "test-claim-254" 68 | allocated["192.168.16.251"] = "test-claim-251" 69 | 70 | checkNoFreeIPError(t, ClaimPool) 71 | } 72 | 73 | func TestIpClaimPoolAvailableIPFromRange(t *testing.T) { 74 | cidr := "192.168.16.248/29" 75 | IPranges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 76 | 77 | allocated := map[string]string{ 78 | "192.168.16.250": "test-claim-250", 79 | "192.168.16.251": "test-claim-251", 80 | } 81 | 82 | expectedIP := "192.168.16.252" 83 | 84 | ClaimPool := &IpClaimPool{ 85 | Spec: IpClaimPoolSpec{ 86 | CIDR: cidr, 87 | Ranges: IPranges, 88 | Allocated: allocated, 89 | }, 90 | } 91 | 92 | checkFreeIP(t, ClaimPool, expectedIP) 93 | 94 | allocated["192.168.16.252"] = "test-claim-252" 95 | 96 | checkNoFreeIPError(t, ClaimPool) 97 | } 98 | 99 | func TestIpClaimPoolRangesProperlyProcessed(t *testing.T) { 100 | cidr := "192.168.16.248/29" 101 | IPranges := [][]string{ 102 | []string{"192.168.16.249", "192.168.16.250"}, 103 | []string{"192.168.16.252", "192.168.16.253"}, 104 | } 105 | 106 | allocated := map[string]string{ 107 | "192.168.16.249": "test-claim-249", 108 | "192.168.16.250": "test-claim-250", 109 | "192.168.16.252": "test-claim-252", 110 | } 111 | 112 | expectedIP := "192.168.16.253" 113 | 114 | ClaimPool := &IpClaimPool{ 115 | Spec: IpClaimPoolSpec{ 116 | CIDR: cidr, 117 | Ranges: IPranges, 118 | Allocated: allocated, 119 | }, 120 | } 121 | 122 | checkFreeIP(t, ClaimPool, expectedIP) 123 | 124 | allocated["192.168.16.253"] = "test-claim-253" 125 | 126 | checkNoFreeIPError(t, ClaimPool) 127 | } 128 | -------------------------------------------------------------------------------- /pkg/controller_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package externalip 16 | 17 | import ( 18 | "strings" 19 | "testing" 20 | "time" 21 | 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 23 | "github.com/stretchr/testify/mock" 24 | 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/pkg/api/v1" 27 | "k8s.io/client-go/tools/cache" 28 | fcache "k8s.io/client-go/tools/cache/testing" 29 | ) 30 | 31 | type fakeIpHandler struct { 32 | mock.Mock 33 | syncer chan struct{} 34 | } 35 | 36 | func (f *fakeIpHandler) Add(iface, cidr string) error { 37 | args := f.Called(iface, cidr) 38 | f.syncer <- struct{}{} 39 | return args.Error(0) 40 | } 41 | 42 | func (f *fakeIpHandler) Del(iface, cidr string) error { 43 | args := f.Called(iface, cidr) 44 | f.syncer <- struct{}{} 45 | return args.Error(0) 46 | } 47 | 48 | func TestControllerServicesAdded(t *testing.T) { 49 | t.Log("started assign ip test") 50 | source := fcache.NewFakeControllerSource() 51 | syncer := make(chan struct{}, 6) 52 | fake := &fakeIpHandler{syncer: syncer} 53 | c := &ExternalIpController{ 54 | Iface: "eth0", 55 | Mask: "24", 56 | source: source, 57 | ipHandler: fake, 58 | Queue: workqueue.NewQueue(), 59 | } 60 | 61 | stopCh := make(chan struct{}) 62 | defer close(stopCh) 63 | go c.Run(stopCh) 64 | 65 | testIps := [][]string{ 66 | {"10.10.0.2", "10.10.0.3"}, 67 | {"10.10.0.2", "10.10.0.3", "10.10.0.4"}, 68 | {"10.10.0.5"}, 69 | } 70 | 71 | added := make(map[string]bool) 72 | for i, ips := range testIps { 73 | for _, ip := range ips { 74 | if _, present := added[ip]; !present { 75 | fake.On("Add", c.Iface, strings.Join([]string{ip, c.Mask}, "/")).Return(nil) 76 | added[ip] = true 77 | } 78 | } 79 | source.Add(&v1.Service{ 80 | ObjectMeta: metav1.ObjectMeta{Name: "service-" + string(i)}, 81 | Spec: v1.ServiceSpec{ExternalIPs: ips}, 82 | }) 83 | } 84 | 85 | for i := 0; i < len(added); i++ { 86 | select { 87 | case <-time.After(200 * time.Millisecond): 88 | t.Errorf("Waiting for calls failed. Current calls %v", fake.Calls) 89 | case <-fake.syncer: 90 | } 91 | } 92 | } 93 | 94 | func TestProcessExternalIps(t *testing.T) { 95 | fake := &fakeIpHandler{syncer: make(chan struct{}, 6)} 96 | c := &ExternalIpController{ 97 | Iface: "eth0", 98 | Mask: "24", 99 | ipHandler: fake, 100 | Queue: workqueue.NewQueue(), 101 | } 102 | testIps := [][]string{ 103 | {"10.10.0.2", "10.10.0.3"}, 104 | {"10.10.0.2", "10.10.0.3", "10.10.0.4"}, 105 | {"10.10.0.5"}, 106 | } 107 | go c.worker() 108 | 109 | added := make(map[string]bool) 110 | for _, ips := range testIps { 111 | for _, ip := range ips { 112 | if _, present := added[ip]; !present { 113 | fake.On("Add", c.Iface, strings.Join([]string{ip, c.Mask}, "/")).Return(nil) 114 | added[ip] = true 115 | } 116 | } 117 | c.processServiceExternalIPs(nil, &v1.Service{Spec: v1.ServiceSpec{ExternalIPs: ips}}, cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)) 118 | } 119 | 120 | for i := 0; i < len(added); i++ { 121 | select { 122 | case <-time.After(200 * time.Millisecond): 123 | t.Errorf("Waiting for calls failed. Current calls %v", fake.Calls) 124 | case <-fake.syncer: 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /pkg/extensions/register.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "fmt" 19 | 20 | "time" 21 | 22 | "strings" 23 | 24 | apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" 25 | apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 26 | "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/client-go/rest" 29 | ) 30 | 31 | var ( 32 | resources = []string{"ip-node", "ip-claim", "ip-claim-pool"} 33 | ) 34 | 35 | func fqName(name string) string { 36 | return fmt.Sprintf("%s.%s", name, GroupName) 37 | } 38 | 39 | func lowercase(name string) string { 40 | parts := strings.Split(name, "-") 41 | return strings.Join(parts, "") 42 | } 43 | 44 | func kind(name string) string { 45 | parts := strings.Split(name, "-") 46 | kindBytes := []byte{} 47 | for _, part := range parts { 48 | kindBytes = append(kindBytes, []byte(strings.Title(part))...) 49 | } 50 | return string(kindBytes) 51 | } 52 | 53 | func EnsureCRDsExist(config *rest.Config) error { 54 | client := apiextensionsclient.NewForConfigOrDie(config) 55 | for _, res := range resources { 56 | if err := createCRD(client, res); err != nil { 57 | return err 58 | } 59 | } 60 | return nil 61 | } 62 | 63 | func RemoveCRDs(config *rest.Config) error { 64 | client := apiextensionsclient.NewForConfigOrDie(config) 65 | for _, res := range resources { 66 | plural := lowercase(res) + "s" 67 | if err := client.Apiextensions().CustomResourceDefinitions().Delete( 68 | fqName(plural), &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { 69 | return err 70 | } 71 | } 72 | return nil 73 | } 74 | 75 | func createCRD(client apiextensionsclient.Interface, name string) error { 76 | singular := lowercase(name) 77 | plural := singular + "s" 78 | crd := &apiextensionsv1beta1.CustomResourceDefinition{ 79 | ObjectMeta: metav1.ObjectMeta{ 80 | Name: fqName(plural), 81 | }, 82 | Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ 83 | Group: GroupName, 84 | Version: Version, 85 | Scope: apiextensionsv1beta1.ClusterScoped, 86 | Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ 87 | Plural: plural, 88 | Singular: singular, 89 | Kind: kind(name), 90 | }, 91 | }, 92 | } 93 | _, err := client.Apiextensions().CustomResourceDefinitions().Create(crd) 94 | if err != nil && !errors.IsAlreadyExists(err) { 95 | return fmt.Errorf("error creating custom resource definition: %v", err) 96 | } 97 | return nil 98 | } 99 | 100 | func WaitCRDsEstablished(config *rest.Config, timeout time.Duration) error { 101 | client := apiextensionsclient.NewForConfigOrDie(config) 102 | interval := time.Tick(200 * time.Millisecond) 103 | timer := time.After(timeout) 104 | for { 105 | select { 106 | case <-timer: 107 | return fmt.Errorf("timed out waiting for CRDs to get established") 108 | case <-interval: 109 | established := 0 110 | for _, res := range resources { 111 | plural := lowercase(res) + "s" 112 | crd, err := client.Apiextensions().CustomResourceDefinitions().Get(fqName(plural), metav1.GetOptions{}) 113 | if err != nil { 114 | break 115 | } 116 | for _, condition := range crd.Status.Conditions { 117 | if condition.Type == apiextensionsv1beta1.Established && 118 | condition.Status == apiextensionsv1beta1.ConditionTrue { 119 | established++ 120 | } 121 | } 122 | } 123 | if established == len(resources) { 124 | return nil 125 | } 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /cmd/app/scheduler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "os" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/scheduler" 23 | 24 | "github.com/golang/glog" 25 | "github.com/spf13/cobra" 26 | "k8s.io/client-go/rest" 27 | "k8s.io/client-go/tools/clientcmd" 28 | "k8s.io/kubernetes/pkg/api" 29 | clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" 30 | "k8s.io/kubernetes/pkg/client/leaderelection" 31 | "k8s.io/kubernetes/pkg/client/leaderelection/resourcelock" 32 | "k8s.io/kubernetes/pkg/client/record" 33 | "k8s.io/kubernetes/pkg/client/restclient" 34 | ) 35 | 36 | func init() { 37 | Root.AddCommand(Scheduler) 38 | } 39 | 40 | var Scheduler = &cobra.Command{ 41 | Aliases: []string{"s"}, 42 | Use: "scheduler", 43 | RunE: func(cmd *cobra.Command, args []string) error { 44 | return InitScheduler() 45 | }, 46 | } 47 | 48 | func InitScheduler() error { 49 | var err error 50 | var config *rest.Config 51 | kubeconfig := AppOpts.Kubeconfig 52 | mask := AppOpts.Mask 53 | config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) 54 | if err != nil { 55 | glog.Errorf("Error parsing config. %v", err) 56 | os.Exit(1) 57 | } 58 | stop := make(chan struct{}) 59 | s, err := scheduler.NewIPClaimScheduler(config, mask, AppOpts.MonitorInterval, AppOpts.NodeFilter) 60 | if err != nil { 61 | glog.Errorf("Crashed during scheduler initialization: %v", err) 62 | os.Exit(2) 63 | } 64 | err = extensions.EnsureCRDsExist(config) 65 | if err != nil { 66 | glog.Fatalf("Crashed while initializing third party resources: %v", err) 67 | } 68 | err = extensions.WaitCRDsEstablished(config, 10*time.Second) 69 | if err != nil { 70 | glog.Fatalf("URLs for tprs are not registered: %v", err) 71 | } 72 | 73 | if !AppOpts.LeaderElection.LeaderElect { 74 | s.Run(stop) 75 | os.Exit(0) 76 | } 77 | glog.V(0).Infof("Running with leader election turned on.") 78 | run := func(_ <-chan struct{}) { 79 | s.Run(stop) 80 | } 81 | 82 | id, err := os.Hostname() 83 | if err != nil { 84 | glog.Fatalf("Cannot get hostname %v", err) 85 | } 86 | 87 | glog.V(0).Infof("Starting scheduler in leader election mode with id=%v", id) 88 | kubernetesConfig, err := restclient.InClusterConfig() 89 | if err != nil { 90 | glog.Fatalf("Error creating config %v", err) 91 | } 92 | leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubernetesConfig, "leader-election")) 93 | if err != nil { 94 | glog.Fatalf("Incorrect configuration %v", err) 95 | } 96 | eventBroadcaster := record.NewBroadcaster() 97 | recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "ipclaim-scheduler"}) 98 | 99 | rl := resourcelock.EndpointsLock{ 100 | EndpointsMeta: api.ObjectMeta{ 101 | Namespace: "kube-system", 102 | Name: "ipclaim-scheduler", 103 | }, 104 | Client: leaderElectionClient, 105 | LockConfig: resourcelock.ResourceLockConfig{ 106 | Identity: id, 107 | EventRecorder: recorder, 108 | }, 109 | } 110 | 111 | leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ 112 | Lock: &rl, 113 | LeaseDuration: AppOpts.LeaderElection.LeaseDuration.Duration, 114 | RenewDeadline: AppOpts.LeaderElection.RenewDeadline.Duration, 115 | RetryPeriod: AppOpts.LeaderElection.RetryPeriod.Duration, 116 | Callbacks: leaderelection.LeaderCallbacks{ 117 | OnStartedLeading: run, 118 | OnStoppedLeading: func() { 119 | glog.Fatalf("lost master") 120 | }, 121 | }, 122 | }) 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /pkg/claimcontroller/controller_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package claimcontroller 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | fclient "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions/testing" 23 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 24 | 25 | "github.com/Mirantis/k8s-externalipcontroller/pkg/utils" 26 | 27 | "github.com/stretchr/testify/assert" 28 | "github.com/stretchr/testify/mock" 29 | "k8s.io/apimachinery/pkg/api/errors" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/runtime/schema" 32 | fcache "k8s.io/client-go/tools/cache/testing" 33 | ) 34 | 35 | type fakeIpHandler struct { 36 | mock.Mock 37 | } 38 | 39 | func (f *fakeIpHandler) Add(iface, cidr string) error { 40 | args := f.Called(iface, cidr) 41 | return args.Error(0) 42 | } 43 | 44 | func (f *fakeIpHandler) Del(iface, cidr string) error { 45 | args := f.Called(iface, cidr) 46 | return args.Error(0) 47 | } 48 | 49 | func TestClaimWatcher(t *testing.T) { 50 | ext := fclient.NewFakeExtClientset() 51 | lw := fcache.NewFakeControllerSource() 52 | queue := workqueue.NewQueue() 53 | fiphandler := &fakeIpHandler{} 54 | defer queue.Close() 55 | stop := make(chan struct{}) 56 | defer close(stop) 57 | c := claimController{ 58 | Uid: "first", 59 | Iface: "eth0", 60 | ExtensionsClientset: ext, 61 | claimSource: lw, 62 | queue: queue, 63 | iphandler: fiphandler, 64 | } 65 | go c.claimWatcher(stop) 66 | go c.worker() 67 | claim := &extensions.IpClaim{ 68 | Metadata: metav1.ObjectMeta{Name: "10.10.0.2-24"}, 69 | Spec: extensions.IpClaimSpec{Cidr: "10.10.0.2/24", NodeName: "first"}, 70 | } 71 | fiphandler.On("Add", c.Iface, claim.Spec.Cidr).Return(nil) 72 | lw.Add(claim) 73 | utils.EventualCondition(t, time.Second*1, func() bool { 74 | return assert.ObjectsAreEqual(1, len(fiphandler.Calls)) 75 | }, "Unexpect calls to iphandler", fiphandler.Calls) 76 | assert.Equal(t, fiphandler.Calls[0].Arguments[0].(string), c.Iface, "Unexpected interface passed to netutils") 77 | assert.Equal(t, fiphandler.Calls[0].Arguments[1].(string), claim.Spec.Cidr, "Unexpected cidr") 78 | 79 | lw.Delete(claim) 80 | fiphandler.On("Del", c.Iface, claim.Spec.Cidr).Return(nil) 81 | utils.EventualCondition(t, time.Second*1, func() bool { 82 | return assert.ObjectsAreEqual(2, len(fiphandler.Calls)) 83 | }, "Unexpect calls to iphandler", fiphandler.Calls) 84 | assert.Equal(t, fiphandler.Calls[1].Arguments[0].(string), c.Iface, "Unexpected interface passed to netutils") 85 | assert.Equal(t, fiphandler.Calls[1].Arguments[1].(string), claim.Spec.Cidr, "Unexpected cidr") 86 | } 87 | 88 | func TestHeartbeatIpNode(t *testing.T) { 89 | ext := fclient.NewFakeExtClientset() 90 | ticker := make(chan time.Time, 3) 91 | for i := 0; i < 3; i++ { 92 | ticker <- time.Time{} 93 | } 94 | stop := make(chan struct{}) 95 | c := claimController{ 96 | Uid: "first", 97 | ExtensionsClientset: ext, 98 | } 99 | qualResource := schema.GroupResource{ 100 | Group: "ipcontroller", 101 | Resource: "ipnode", 102 | } 103 | ipnode := &extensions.IpNode{ 104 | Metadata: metav1.ObjectMeta{Name: c.Uid}, 105 | } 106 | ext.Ipnodes.On("Get", c.Uid).Return(&extensions.IpNode{}, errors.NewNotFound(qualResource, c.Uid)) 107 | ext.Ipnodes.On("Create", mock.Anything).Return(nil) 108 | ext.Ipnodes.On("Get", c.Uid).Return(ipnode, nil).Twice() 109 | ext.Ipnodes.On("Update", mock.Anything).Return(nil).Twice() 110 | go c.heartbeatIpNode(stop, ticker) 111 | utils.EventualCondition(t, time.Second*1, func() bool { 112 | return assert.ObjectsAreEqual(6, len(ext.Ipnodes.Calls)) 113 | }, "Unexpect calls to iphandler", ext.Ipnodes.Calls) 114 | } 115 | -------------------------------------------------------------------------------- /pkg/netutils/netutils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package netutils 16 | 17 | import ( 18 | "net" 19 | 20 | "github.com/golang/glog" 21 | "github.com/vishvananda/netlink" 22 | "github.com/google/gopacket" 23 | "github.com/google/gopacket/layers" 24 | "github.com/google/gopacket/pcap" 25 | ) 26 | 27 | func writeARP(handle *pcap.Handle, iface *net.Interface, addr *net.IPNet) error { 28 | // Set up all the layers' fields we can. 29 | eth := layers.Ethernet{ 30 | SrcMAC: iface.HardwareAddr, 31 | DstMAC: net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, 32 | EthernetType: layers.EthernetTypeARP, 33 | } 34 | arp := layers.ARP{ 35 | AddrType: layers.LinkTypeEthernet, 36 | Protocol: layers.EthernetTypeIPv4, 37 | HwAddressSize: 6, 38 | ProtAddressSize: 4, 39 | Operation: layers.ARPRequest, 40 | SourceHwAddress: []byte(iface.HardwareAddr), 41 | SourceProtAddress: []byte(addr.IP.To4()), 42 | DstHwAddress: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, 43 | DstProtAddress: []byte(addr.IP.To4()), 44 | } 45 | // Set up buffer and options for serialization. 46 | buf := gopacket.NewSerializeBuffer() 47 | opts := gopacket.SerializeOptions{ 48 | FixLengths: true, 49 | ComputeChecksums: false, 50 | } 51 | // Send one packet for every address. 52 | gopacket.SerializeLayers(buf, opts, ð, &arp) 53 | if err := handle.WritePacketData(buf.Bytes()); err != nil { 54 | return err 55 | } 56 | return nil 57 | } 58 | 59 | func ArpAnnouncement(ifname string, addr *net.IPNet) error { 60 | iface, err := net.InterfaceByName(ifname) 61 | if err != nil { 62 | return err 63 | } 64 | handle, err := pcap.OpenLive(iface.Name, 65536, true, pcap.BlockForever) 65 | if err != nil { 66 | return err 67 | } 68 | defer handle.Close() 69 | return writeARP(handle, iface, addr) 70 | } 71 | 72 | // EnsureIPAssigned will check if ip is already present on a given link 73 | func EnsureIPAssigned(iface, cidr string) error { 74 | link, err := netlink.LinkByName(iface) 75 | if err != nil { 76 | return err 77 | } 78 | addr, err := netlink.ParseAddr(cidr) 79 | if err != nil { 80 | return err 81 | } 82 | addrList, err := netlink.AddrList(link, netlink.FAMILY_ALL) 83 | if err != nil { 84 | return err 85 | } 86 | for i := range addrList { 87 | if addrList[i].IPNet.String() == addr.IPNet.String() { 88 | return nil 89 | } 90 | } 91 | err = netlink.AddrAdd(link, addr) 92 | if err != nil { 93 | return err 94 | } 95 | if iface != "lo" { 96 | return ArpAnnouncement(iface, addr.IPNet) 97 | } 98 | return nil 99 | } 100 | 101 | // EnsureIPUnassigned ensure that given IP is not present on a given link 102 | func EnsureIPUnassigned(iface, cidr string) error { 103 | link, err := netlink.LinkByName(iface) 104 | if err != nil { 105 | return err 106 | } 107 | addr, err := netlink.ParseAddr(cidr) 108 | if err != nil { 109 | return err 110 | } 111 | addrList, err := netlink.AddrList(link, netlink.FAMILY_ALL) 112 | if err != nil { 113 | return err 114 | } 115 | for i := range addrList { 116 | if addrList[i].IPNet.String() == addr.IPNet.String() { 117 | return netlink.AddrDel(link, addr) 118 | } 119 | } 120 | return nil 121 | } 122 | 123 | type IPHandler interface { 124 | Add(iface, cidr string) error 125 | Del(iface, cidr string) error 126 | } 127 | 128 | type LinuxIPHandler struct{} 129 | 130 | func (l LinuxIPHandler) Add(iface, cidr string) error { 131 | glog.V(2).Infof("Adding addr %v on link %v", cidr, iface) 132 | return EnsureIPAssigned(iface, cidr) 133 | } 134 | func (l LinuxIPHandler) Del(iface, cidr string) error { 135 | glog.V(2).Infof("Removing addr %v from link %v", cidr, iface) 136 | return EnsureIPUnassigned(iface, cidr) 137 | } 138 | 139 | type AddCIDR struct { 140 | Cidr string 141 | } 142 | 143 | type DelCIDR struct { 144 | Cidr string 145 | } 146 | 147 | func IPIncrement(ip net.IP) { 148 | for j := len(ip) - 1; j >= 0; j-- { 149 | ip[j]++ 150 | if ip[j] > 0 { 151 | break 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /pkg/extensions/testing/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package testing 16 | 17 | import ( 18 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 19 | 20 | "github.com/stretchr/testify/mock" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/watch" 23 | ) 24 | 25 | type FakeExtClientset struct { 26 | mock.Mock 27 | Ipclaims *fakeIpClaims 28 | Ipnodes *fakeIpNodes 29 | Ipclaimpools *fakeIpClaimPools 30 | } 31 | 32 | func (f *FakeExtClientset) IPClaims() extensions.IPClaimsInterface { 33 | return f.Ipclaims 34 | } 35 | 36 | func (f *FakeExtClientset) IPNodes() extensions.IPNodesInterface { 37 | return f.Ipnodes 38 | } 39 | 40 | func (f *FakeExtClientset) IPClaimPools() extensions.IPClaimPoolsInterface { 41 | return f.Ipclaimpools 42 | } 43 | 44 | type fakeIpClaims struct { 45 | mock.Mock 46 | } 47 | 48 | func (f *fakeIpClaims) Get(name string) (*extensions.IpClaim, error) { 49 | args := f.Called(name) 50 | return args.Get(0).(*extensions.IpClaim), args.Error(1) 51 | } 52 | 53 | func (f *fakeIpClaims) Create(ipclaim *extensions.IpClaim) (*extensions.IpClaim, error) { 54 | args := f.Called(ipclaim) 55 | return ipclaim, args.Error(0) 56 | } 57 | 58 | func (f *fakeIpClaims) List(opts metav1.ListOptions) (*extensions.IpClaimList, error) { 59 | args := f.Called(opts) 60 | return args.Get(0).(*extensions.IpClaimList), args.Error(1) 61 | } 62 | 63 | func (f *fakeIpClaims) Update(ipclaim *extensions.IpClaim) (*extensions.IpClaim, error) { 64 | args := f.Called(ipclaim) 65 | return ipclaim, args.Error(0) 66 | } 67 | 68 | func (f *fakeIpClaims) Delete(name string, opts *metav1.DeleteOptions) error { 69 | args := f.Called(name, opts) 70 | return args.Error(0) 71 | } 72 | 73 | func (f *fakeIpClaims) Watch(_ metav1.ListOptions) (watch.Interface, error) { 74 | return nil, nil 75 | } 76 | 77 | type fakeIpNodes struct { 78 | mock.Mock 79 | } 80 | 81 | func (f *fakeIpNodes) Create(ipnode *extensions.IpNode) (*extensions.IpNode, error) { 82 | args := f.Called(ipnode) 83 | return ipnode, args.Error(0) 84 | } 85 | 86 | func (f *fakeIpNodes) List(opts metav1.ListOptions) (*extensions.IpNodeList, error) { 87 | args := f.Called(opts) 88 | return args.Get(0).(*extensions.IpNodeList), args.Error(1) 89 | } 90 | 91 | func (f *fakeIpNodes) Update(ipnode *extensions.IpNode) (*extensions.IpNode, error) { 92 | args := f.Called(ipnode) 93 | return args.Get(0).(*extensions.IpNode), args.Error(1) 94 | } 95 | 96 | func (f *fakeIpNodes) Delete(name string, opts *metav1.DeleteOptions) error { 97 | args := f.Called(name, opts) 98 | return args.Error(0) 99 | } 100 | 101 | func (f *fakeIpNodes) Get(name string) (*extensions.IpNode, error) { 102 | args := f.Called(name) 103 | return args.Get(0).(*extensions.IpNode), args.Error(1) 104 | } 105 | 106 | func (f *fakeIpNodes) Watch(_ metav1.ListOptions) (watch.Interface, error) { 107 | return nil, nil 108 | } 109 | 110 | type fakeIpClaimPools struct { 111 | mock.Mock 112 | } 113 | 114 | func (f *fakeIpClaimPools) Create(ipclaimpool *extensions.IpClaimPool) (*extensions.IpClaimPool, error) { 115 | args := f.Called(ipclaimpool) 116 | return ipclaimpool, args.Error(0) 117 | } 118 | 119 | func (f *fakeIpClaimPools) List(opts metav1.ListOptions) (*extensions.IpClaimPoolList, error) { 120 | args := f.Called(opts) 121 | return args.Get(0).(*extensions.IpClaimPoolList), args.Error(1) 122 | } 123 | 124 | func (f *fakeIpClaimPools) Update(ipclaimpool *extensions.IpClaimPool) (*extensions.IpClaimPool, error) { 125 | args := f.Called(ipclaimpool) 126 | return args.Get(0).(*extensions.IpClaimPool), args.Error(1) 127 | } 128 | 129 | func (f *fakeIpClaimPools) Delete(name string, opts *metav1.DeleteOptions) error { 130 | args := f.Called(name, opts) 131 | return args.Error(0) 132 | } 133 | 134 | func (f *fakeIpClaimPools) Get(name string) (*extensions.IpClaimPool, error) { 135 | args := f.Called(name) 136 | return args.Get(0).(*extensions.IpClaimPool), args.Error(1) 137 | } 138 | 139 | func NewFakeExtClientset() *FakeExtClientset { 140 | return &FakeExtClientset{ 141 | Ipclaims: &fakeIpClaims{}, 142 | Ipnodes: &fakeIpNodes{}, 143 | Ipclaimpools: &fakeIpClaimPools{}, 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | IMAGE_REPO ?= mirantis/k8s-externalipcontroller 2 | IMAGE_TAG ?= latest 3 | DOCKER_BUILD ?= no 4 | 5 | BUILD_DIR = _output 6 | VENDOR_DIR = vendor 7 | ROOT_DIR = $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) 8 | 9 | ENV_PREPARE_MARKER = .env.complete 10 | BUILD_IMAGE_MARKER = .build-image.complete 11 | K8S_VERSION = v1.7 12 | 13 | ifeq ($(DOCKER_BUILD), yes) 14 | _DOCKER_GOPATH = /go 15 | _DOCKER_WORKDIR = $(_DOCKER_GOPATH)/src/github.com/Mirantis/k8s-externalipcontroller/ 16 | _DOCKER_IMAGE = golang:1.7 17 | DOCKER_DEPS = apt-get update; apt-get install -y libpcap-dev; 18 | DOCKER_EXEC = docker run --rm -it -v "$(ROOT_DIR):$(_DOCKER_WORKDIR)" \ 19 | -w "$(_DOCKER_WORKDIR)" $(_DOCKER_IMAGE) 20 | else 21 | DOCKER_EXEC = 22 | DOCKER_DEPS = 23 | endif 24 | 25 | .PHONY: help 26 | help: 27 | @echo "Usage: 'make '" 28 | @echo "" 29 | @echo "Targets:" 30 | @echo "help - Print this message and exit" 31 | @echo "get-deps - Install project dependencies" 32 | @echo "containerized-build - Build ipmanager binary in container" 33 | @echo "build - Build ipmanager binary" 34 | @echo "build-image - Build docker image" 35 | @echo "test - Run all tests" 36 | @echo "unit - Run unit tests" 37 | @echo "integration - Run integration tests" 38 | @echo "e2e - Run e2e tests" 39 | @echo "clean - Delete binaries" 40 | @echo "clean-all - Delete binaries and vendor files" 41 | 42 | .PHONY: get-deps 43 | get-deps: $(VENDOR_DIR) 44 | 45 | 46 | .PHONY: build 47 | build: $(BUILD_DIR)/ipmanager 48 | 49 | 50 | .PHONY: containerized-build 51 | containerized-build: 52 | make build DOCKER_BUILD=yes 53 | 54 | 55 | .PHONY: build-image 56 | build-image docker: $(BUILD_IMAGE_MARKER) 57 | 58 | 59 | .PHONY: unit 60 | unit: 61 | $(DOCKER_EXEC) bash -xc '$(DOCKER_DEPS) \ 62 | go test -v ./pkg/...' 63 | 64 | 65 | .PHONY: integration 66 | integration: $(BUILD_DIR)/integration.test $(ENV_PREPARE_MARKER) 67 | sudo $(BUILD_DIR)/integration.test --ginkgo.v --logtostderr --v=10 68 | 69 | 70 | .PHONY: e2e 71 | e2e: $(BUILD_DIR)/e2e.test $(ENV_PREPARE_MARKER) run-e2e 72 | 73 | run-e2e: 74 | sudo $(BUILD_DIR)/e2e.test --master=http://localhost:8080 \ 75 | --testlink=br-$(shell docker network ls -f name=kubeadm-dind-net -q) -ginkgo.v -ginkgo.skip=Flaky 76 | 77 | 78 | .PHONY: test 79 | test: unit integration e2e 80 | 81 | 82 | .PHONY: clean 83 | clean: 84 | rm -rf $(BUILD_DIR) 85 | 86 | 87 | .PHONY: clean-all 88 | clean-all: clean 89 | rm -rf $(VENDOR_DIR) 90 | rm -f $(BUILD_IMAGE_MARKER) 91 | docker rmi -f $(IMAGE_REPO):$(IMAGE_TAG) 92 | 93 | 94 | $(BUILD_DIR): 95 | mkdir -p $(BUILD_DIR) 96 | 97 | 98 | $(BUILD_DIR)/ipmanager: $(BUILD_DIR) $(VENDOR_DIR) 99 | $(DOCKER_EXEC) bash -xc '$(DOCKER_DEPS) \ 100 | go build --ldflags "-extldflags \"-static\"" \ 101 | -o $@ ./cmd/ipmanager/ ; \ 102 | chown $(shell id -u):$(shell id -g) -R _output' 103 | 104 | 105 | $(BUILD_DIR)/e2e.test: $(BUILD_DIR) $(VENDOR_DIR) 106 | $(DOCKER_EXEC) bash -xc '$(DOCKER_DEPS) \ 107 | go test -c -o $@ ./test/e2e/' 108 | 109 | 110 | $(BUILD_DIR)/integration.test: $(BUILD_DIR) $(VENDOR_DIR) 111 | $(DOCKER_EXEC) bash -xc '$(DOCKER_DEPS) \ 112 | go test -c -o $@ ./test/integration/' 113 | 114 | 115 | $(BUILD_IMAGE_MARKER): $(BUILD_DIR)/ipmanager 116 | docker build -t $(IMAGE_REPO):$(IMAGE_TAG) . 117 | echo > $(BUILD_IMAGE_MARKER) 118 | 119 | 120 | $(VENDOR_DIR): 121 | $(DOCKER_EXEC) bash -xc 'go get github.com/Masterminds/glide && \ 122 | glide install --strip-vendor; \ 123 | chown $(shell id -u):$(shell id -g) -R vendor' 124 | 125 | .PHONY: build-env 126 | build-env: kubeadm-dind-cluster $(BUILD_IMAGE_MARKER) 127 | docker save $(IMAGE_REPO):$(IMAGE_TAG) -o $(BUILD_DIR)/ipcontroller.tar 128 | docker cp $(BUILD_DIR)/ipcontroller.tar kube-master:/ 129 | docker exec -ti kube-master docker load -i /ipcontroller.tar 130 | docker exec -ti kube-master ip l set dev dind0 promisc on 131 | docker cp $(BUILD_DIR)/ipcontroller.tar kube-node-1:/ 132 | docker exec -ti kube-node-1 docker load -i /ipcontroller.tar 133 | docker exec -ti kube-node-1 ip l set dev dind0 promisc on 134 | ~/.kubeadm-dind-cluster/kubectl label node kube-node-1 --overwrite ipcontroller= 135 | docker cp $(BUILD_DIR)/ipcontroller.tar kube-node-2:/ 136 | docker exec -ti kube-node-2 docker load -i /ipcontroller.tar 137 | docker exec -ti kube-node-2 ip l set dev dind0 promisc on 138 | ~/.kubeadm-dind-cluster/kubectl label node kube-node-2 --overwrite ipcontroller= 139 | 140 | $(ENV_PREPARE_MARKER): build-env 141 | touch $(ENV_PREPARE_MARKER) 142 | 143 | .PHONY: clean-k8s 144 | clean-k8s: 145 | -./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh clean 146 | -rm -rf kubeadm-dind-cluster/ 147 | 148 | kubeadm-dind-cluster: 149 | git clone https://github.com/Mirantis/kubeadm-dind-cluster.git 150 | ./kubeadm-dind-cluster/fixed/dind-cluster-$(K8S_VERSION).sh up 151 | -------------------------------------------------------------------------------- /doc/operating-modes.md: -------------------------------------------------------------------------------- 1 | Basic Modules and Operating Modes 2 | ================================= 3 | 4 | ## Basic Modules 5 | 6 | The application consists of two basic modules: controller and scheduler. 7 | 8 | Controller module manages IPs on its node (brings up, deletes, ensures that list 9 | of IPs on node reflects the list of IPs required for services). 10 | Controller(s) should be run on nodes that have external connectivity as 11 | External IPs will be spawned on those nodes. Each node should have no more than 12 | one controller at a time. 13 | 14 | Scheduler module processes IP claims from services and distributes them among 15 | the controllers (i.e. nodes). 16 | Scheduler(s) can be run on any nodes as they just schedule claims among the 17 | controllers. Scheduler module is not obligatory to be run. It is not in use 18 | while the application runs in Simple mode. 19 | 20 | ## Operating Modes 21 | 22 | External IP Controller application may be run in one of the operating modes: 23 | * [Simple](#simple-mode) 24 | * [Claims](#claims-mode) 25 | 26 | ## Simple Mode 27 | 28 | External IP controller application (its controller module) will be run on one of 29 | the nodes. It should be run on node that has external connectivity as all the 30 | External IPs will be spawned on that node. 31 | Kubernetes provides fail-over for External IP controller application. So, when 32 | there's a problem with k8s node, External IP controller will be spawned on 33 | another k8s worker node and will bring External IPs up on that node. 34 | Simple mode is easy to setup and takes fewer resources. It makes sense when all 35 | IPs should be brought up on the same node. However, fail-over in this mode takes 36 | longer than in Claims mode (k8s detects node failure in much longer intervals by 37 | default, this could be optimized - see the ``Fail-Over Optimization`` document) 38 | and it may work wrong in some cases. 39 | Please also refer to the ``Simple Deployment Scheme`` for more information. 40 | 41 | # Parameters 42 | 43 | Next command-line parameters are available in Simple mode for controller module: 44 | * `iface` - interface that will be used to assign IP addresses (default "eth0"). 45 | * `kubeconfig` - kubeconfig to use with kubernetes client (default ""; incluster 46 | configuration for auth will be used by default). 47 | * `mask` - mask part of network CIDR (default "32"). 48 | * `resync` - interval to resync state for all ips (default 20 sec). 49 | It is usually enough to set `iface` and `mask` parameters. 50 | 51 | ## Claims Mode 52 | 53 | External IP controller application will be run on several nodes. One or more 54 | controller modules and one or more scheduler modules will be run. Controller 55 | modules should be run on nodes where IPs are expected to be spawned. Scheduler 56 | modules can be run on any nodes. There is no much sense to run more than one 57 | scheduler on every particular node. Several scheduler modules are run to provide 58 | HA for scheduler (A/B mode). So that in case of no response from the active 59 | scheduler (e.g. on node failure) another one becomes active. If more than one 60 | scheduler will be used then scheduler election mode should be switched on 61 | (parameter `leader-elect=true`) otherwise there can be race conditions between 62 | schedulers. 63 | It is better not to run both scheduler and controller on the same node because 64 | in case of node outage IPs rescheduling will take more time. 65 | 66 | # IPs Distribution in Claims Mode 67 | 68 | There can be different rules of IPs distribution among controllers (i.e. nodes) 69 | in Claims mode. This is controlled by `nodefilter` parameter. The default rule 70 | is to distribute IPs evenly among all the controllers (`nodefilter=fair`). 71 | An alternative rule is `nodefilter=first-alive` where all IPs will be spawned 72 | on the first available controller (i.e. node). Claims mode with the `first-alive` 73 | rule is similar to Simple mode but with more responsive and correct fail-over. 74 | 75 | # Parameters 76 | 77 | Next command-line parameters are available in Claims mode for controller module: 78 | * `iface` - interface that will be used to assign IP addresses (default "eth0"). 79 | * `hb` - how often to send heartbeats from controllers (default 2 sec). 80 | * `kubeconfig` - kubeconfig to use with kubernetes client (default ""; incluster 81 | configuration for auth will be used by default). 82 | * `resync` - interval to resync state for all IPs (default 20 sec). 83 | * `hostname` - use provided hostname instead of os.Hostname (default 84 | os.Hostname). 85 | 86 | Next command-line parameters are available in Claims mode for scheduler module: 87 | * `kubeconfig` - kubeconfig to use with a kubernetes client (default ""; 88 | incluster configuration for authentication will be used by default). 89 | * `mask` - mask part of network CIDR (default "32"), it is not in use for 90 | auto-allocation. 91 | * `nodefilter` - node filter to use while dispatching IP claims; it controls IPs 92 | distribution between controllers (default "fair"). 93 | * `monitor` - how often to check controllers responsiveness (default 4 94 | sec). 95 | * `leader-elect` - switch on the leader election mechanism for scheduler modules. 96 | Other leader election parameters are also taken into account. Please refer to 97 | kubernetes documentation for more detail. 98 | -------------------------------------------------------------------------------- /test/integration/network_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | "strings" 21 | "time" 22 | 23 | controller "github.com/Mirantis/k8s-externalipcontroller/pkg" 24 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 25 | 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/client-go/pkg/api/v1" 28 | fcache "k8s.io/client-go/tools/cache/testing" 29 | 30 | "github.com/vishvananda/netlink" 31 | 32 | "reflect" 33 | 34 | . "github.com/onsi/ginkgo" 35 | . "github.com/onsi/gomega" 36 | ) 37 | 38 | var _ = Describe("Network [sudo]", func() { 39 | 40 | var linkNames []string 41 | 42 | BeforeEach(func() { 43 | linkNames = []string{"test11", "test12"} 44 | ensureLinksRemoved(linkNames...) 45 | }) 46 | 47 | AfterEach(func() { 48 | ensureLinksRemoved(linkNames...) 49 | }) 50 | 51 | It("Multiple ips can be assigned", func() { 52 | link := &netlink.Dummy{netlink.LinkAttrs{Name: linkNames[0]}} 53 | By("adding dummy link with name " + link.Attrs().Name) 54 | Expect(netlink.LinkAdd(link)).NotTo(HaveOccurred()) 55 | By("getting link up") 56 | Expect(netlink.LinkSetUp(link)).NotTo(HaveOccurred()) 57 | cidrToAssign := []string{"10.10.0.2/24", "10.10.0.2/24", "10.10.0.3/24"} 58 | for _, cidr := range cidrToAssign { 59 | err := netutils.EnsureIPAssigned(link.Attrs().Name, cidr) 60 | Expect(err).NotTo(HaveOccurred()) 61 | } 62 | addrList, err := netlink.AddrList(link, netlink.FAMILY_V4) 63 | Expect(err).NotTo(HaveOccurred()) 64 | ipSet := map[string]bool{} 65 | expectedIpSet := map[string]bool{"10.10.0.2/24": true, "10.10.0.3/24": true} 66 | for i := range addrList { 67 | ipSet[addrList[i].IPNet.String()] = true 68 | } 69 | Expect(expectedIpSet).To(BeEquivalentTo(ipSet)) 70 | }) 71 | 72 | It("Controller with noop manager will create provided externalIPs", func() { 73 | link := &netlink.Dummy{netlink.LinkAttrs{Name: linkNames[0]}} 74 | 75 | By("adding link for controller") 76 | Expect(netlink.LinkAdd(link)).NotTo(HaveOccurred()) 77 | Expect(netlink.LinkSetUp(link)).NotTo(HaveOccurred()) 78 | 79 | By("creating and running controller with fake source") 80 | stop := make(chan struct{}) 81 | defer close(stop) 82 | source := fcache.NewFakeControllerSource() 83 | c := controller.NewExternalIpControllerWithSource("1", link.Attrs().Name, "24", source) 84 | go c.Run(stop) 85 | 86 | testIps := [][]string{ 87 | {"10.10.0.2", "10.10.0.3"}, 88 | {"10.10.0.2", "10.10.0.3", "10.10.0.4"}, 89 | {"10.10.0.5"}, 90 | } 91 | services := map[string]*v1.Service{} 92 | expectedIps := map[string]bool{} 93 | for i, ips := range testIps { 94 | for _, ip := range ips { 95 | expectedIps[strings.Join([]string{ip, c.Mask}, "/")] = true 96 | } 97 | 98 | svc := &v1.Service{ 99 | ObjectMeta: metav1.ObjectMeta{Name: "service-" + strconv.Itoa(i)}, 100 | Spec: v1.ServiceSpec{ExternalIPs: ips}, 101 | } 102 | services[svc.Name] = svc 103 | source.Add(svc) 104 | } 105 | By("waiting until ips will be assigned") 106 | verifyAddrs(link, expectedIps) 107 | By("updating service with new ip list and waiting until ips assignment will be updated") 108 | svcToUpdate := services["service-2"] 109 | svcToUpdate.Spec.ExternalIPs = []string{"10.10.0.7"} 110 | source.Modify(svcToUpdate) 111 | expectedIps["10.10.0.7/24"] = true 112 | delete(expectedIps, "10.10.0.5/24") 113 | verifyAddrs(link, expectedIps) 114 | By("removing service with single ip and waiting until this ip won't be on a link") 115 | source.Delete(services["service-2"]) 116 | delete(expectedIps, "10.10.0.7/24") 117 | verifyAddrs(link, expectedIps) 118 | By("removing service with ips that are assigned to some other service and confirming that they are still on link") 119 | source.Delete(services["service-1"]) 120 | delete(expectedIps, "10.10.0.4/24") 121 | verifyAddrs(link, expectedIps) 122 | }) 123 | }) 124 | 125 | func verifyAddrs(link netlink.Link, expectedIps map[string]bool) { 126 | Eventually(func() error { 127 | resultIps := map[string]bool{} 128 | addrList, err := netlink.AddrList(link, netlink.FAMILY_V4) 129 | if err != nil { 130 | return err 131 | } 132 | for _, addr := range addrList { 133 | resultIps[addr.IPNet.String()] = true 134 | } 135 | if !reflect.DeepEqual(expectedIps, resultIps) { 136 | return fmt.Errorf("Assigned ips %v are not equal to expected %v.", resultIps, expectedIps) 137 | } 138 | return nil 139 | }, 20*time.Second, 1*time.Second).Should(BeNil()) 140 | } 141 | 142 | func ensureLinksRemoved(links ...string) { 143 | for _, l := range links { 144 | link, err := netlink.LinkByName(l) 145 | if err != nil { 146 | continue 147 | } 148 | netlink.LinkDel(link) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /test/e2e/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "bytes" 19 | "flag" 20 | "fmt" 21 | "io" 22 | "net/url" 23 | "strings" 24 | "time" 25 | 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | apiremotecommand "k8s.io/apimachinery/pkg/util/remotecommand" 28 | "k8s.io/client-go/kubernetes" 29 | "k8s.io/client-go/pkg/api" 30 | _ "k8s.io/client-go/pkg/api/install" 31 | "k8s.io/client-go/pkg/api/v1" 32 | v1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1" 33 | "k8s.io/client-go/rest" 34 | "k8s.io/client-go/tools/clientcmd" 35 | "k8s.io/client-go/tools/remotecommand" 36 | 37 | . "github.com/onsi/ginkgo" 38 | . "github.com/onsi/gomega" 39 | "k8s.io/apimachinery/pkg/api/errors" 40 | ) 41 | 42 | var ( 43 | MASTER string 44 | TESTLINK string 45 | MASTERNAME string 46 | ) 47 | 48 | func init() { 49 | flag.StringVar(&MASTER, "master", "http://apiserver:8888", "apiserver address to use with restclient") 50 | flag.StringVar(&TESTLINK, "testlink", "eth0", "link to use on the side of tests") 51 | flag.StringVar(&MASTERNAME, "mastername", "kube-master", "node that wont be used for scheduling") 52 | } 53 | 54 | func GetTestLink() string { 55 | return TESTLINK 56 | } 57 | 58 | func Logf(format string, a ...interface{}) { 59 | fmt.Fprintf(GinkgoWriter, format, a...) 60 | } 61 | 62 | func LoadConfig() *rest.Config { 63 | config, err := clientcmd.BuildConfigFromFlags(MASTER, "") 64 | Expect(err).NotTo(HaveOccurred()) 65 | return config 66 | } 67 | 68 | func KubeClient() (*kubernetes.Clientset, error) { 69 | Logf("Using master %v\n", MASTER) 70 | config := LoadConfig() 71 | clientset, err := kubernetes.NewForConfig(config) 72 | Expect(err).NotTo(HaveOccurred()) 73 | return clientset, nil 74 | } 75 | 76 | func WaitForReady(clientset *kubernetes.Clientset, pod *v1.Pod) { 77 | Eventually(func() error { 78 | podUpdated, err := clientset.Core().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) 79 | if err != nil { 80 | return err 81 | } 82 | if podUpdated.Status.Phase != v1.PodRunning { 83 | return fmt.Errorf("pod %v is not running phase: %v", podUpdated.Name, podUpdated.Status.Phase) 84 | } 85 | return nil 86 | }, 120*time.Second, 5*time.Second).Should(BeNil()) 87 | } 88 | 89 | func DumpLogs(clientset *kubernetes.Clientset, pods ...v1.Pod) { 90 | for _, pod := range pods { 91 | dumpLogs(clientset, pod) 92 | } 93 | } 94 | 95 | func dumpLogs(clientset *kubernetes.Clientset, pod v1.Pod) { 96 | req := clientset.Core().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}) 97 | readCloser, err := req.Stream() 98 | Expect(err).NotTo(HaveOccurred()) 99 | defer readCloser.Close() 100 | Logf("\n Dumping logs for %v:%v \n", pod.Namespace, pod.Name) 101 | _, err = io.Copy(GinkgoWriter, readCloser) 102 | Expect(err).NotTo(HaveOccurred()) 103 | } 104 | 105 | func ExecInPod(clientset *kubernetes.Clientset, pod v1.Pod, cmd ...string) (string, string, error) { 106 | Logf("Running %v in %v\n", cmd, pod.Name) 107 | container := pod.Spec.Containers[0].Name 108 | var stdout, stderr bytes.Buffer 109 | config := LoadConfig() 110 | rest := clientset.Core().RESTClient() 111 | req := rest.Post(). 112 | Resource("pods"). 113 | Name(pod.Name). 114 | Namespace(pod.Namespace). 115 | SubResource("exec"). 116 | Param("container", container) 117 | req.VersionedParams(&api.PodExecOptions{ 118 | Container: container, 119 | Command: cmd, 120 | TTY: false, 121 | Stdin: false, 122 | Stdout: true, 123 | Stderr: true, 124 | }, api.ParameterCodec) 125 | err := execute("POST", req.URL(), config, nil, &stdout, &stderr, false) 126 | Logf("Error %v: %v\n", cmd, stderr.String()) 127 | Logf("Output %v: %v\n", cmd, stdout.String()) 128 | return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err 129 | } 130 | 131 | func execute(method string, url *url.URL, config *rest.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { 132 | exec, err := remotecommand.NewExecutor(config, method, url) 133 | if err != nil { 134 | return err 135 | } 136 | return exec.Stream(remotecommand.StreamOptions{ 137 | SupportedProtocols: []string{ 138 | apiremotecommand.StreamProtocolV4Name, 139 | apiremotecommand.StreamProtocolV3Name, 140 | apiremotecommand.StreamProtocolV2Name, 141 | apiremotecommand.StreamProtocolV1Name, 142 | }, 143 | Stdin: stdin, 144 | Stdout: stdout, 145 | Stderr: stderr, 146 | Tty: tty, 147 | }) 148 | } 149 | 150 | // AddServiceAccountToAdmins will add system:serviceaccounts to cluster-admin ClusterRole 151 | func AddServiceAccountToAdmins(c kubernetes.Interface) { 152 | By("Adding service account group to cluster-admin role") 153 | roleBinding := &v1beta1.ClusterRoleBinding{ 154 | ObjectMeta: metav1.ObjectMeta{ 155 | Name: "system:serviceaccount-admin", 156 | }, 157 | Subjects: []v1beta1.Subject{{ 158 | Kind: "Group", 159 | Name: "system:serviceaccounts", 160 | }}, 161 | RoleRef: v1beta1.RoleRef{ 162 | Kind: "ClusterRole", 163 | Name: "cluster-admin", 164 | APIGroup: "rbac.authorization.k8s.io", 165 | }, 166 | } 167 | _, err := c.Rbac().ClusterRoleBindings().Create(roleBinding) 168 | if !errors.IsAlreadyExists(err) { 169 | Expect(err).NotTo(HaveOccurred(), "Failed to create role binding for serviceaccounts") 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /pkg/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package externalip 16 | 17 | import ( 18 | "reflect" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 23 | "github.com/golang/glog" 24 | 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/apimachinery/pkg/watch" 28 | "k8s.io/client-go/kubernetes" 29 | "k8s.io/client-go/pkg/api" 30 | "k8s.io/client-go/pkg/api/v1" 31 | "k8s.io/client-go/rest" 32 | "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | type ExternalIpController struct { 36 | Uid string 37 | Iface string 38 | Mask string 39 | 40 | source cache.ListerWatcher 41 | ipHandler netutils.IPHandler 42 | Queue workqueue.QueueType 43 | 44 | resyncInterval time.Duration 45 | } 46 | 47 | func NewExternalIpController(config *rest.Config, uid, iface, mask string, resyncInterval time.Duration) (*ExternalIpController, error) { 48 | clientset, err := kubernetes.NewForConfig(config) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | lw := &cache.ListWatch{ 54 | ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 55 | return clientset.Core().Services(api.NamespaceAll).List(metav1.ListOptions{}) 56 | }, 57 | WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 58 | return clientset.Core().Services(api.NamespaceAll).Watch(metav1.ListOptions{}) 59 | }, 60 | } 61 | return &ExternalIpController{ 62 | Uid: uid, 63 | Iface: iface, 64 | Mask: mask, 65 | source: lw, 66 | ipHandler: netutils.LinuxIPHandler{}, 67 | Queue: workqueue.NewQueue(), 68 | resyncInterval: resyncInterval, 69 | }, nil 70 | } 71 | 72 | func NewExternalIpControllerWithSource(uid, iface, mask string, source cache.ListerWatcher) *ExternalIpController { 73 | return &ExternalIpController{ 74 | Uid: uid, 75 | Iface: iface, 76 | Mask: mask, 77 | source: source, 78 | ipHandler: netutils.LinuxIPHandler{}, 79 | Queue: workqueue.NewQueue(), 80 | } 81 | } 82 | 83 | func (c *ExternalIpController) Run(stopCh chan struct{}) { 84 | glog.Infof("Starting externalipcontroller") 85 | var store cache.Store 86 | store, controller := cache.NewInformer( 87 | c.source, 88 | &v1.Service{}, 89 | c.resyncInterval, 90 | cache.ResourceEventHandlerFuncs{ 91 | AddFunc: func(obj interface{}) { 92 | c.processServiceExternalIPs(nil, obj.(*v1.Service), store) 93 | }, 94 | UpdateFunc: func(old, cur interface{}) { 95 | c.processServiceExternalIPs(old.(*v1.Service), cur.(*v1.Service), store) 96 | }, 97 | DeleteFunc: func(obj interface{}) { 98 | c.processServiceExternalIPs(obj.(*v1.Service), nil, store) 99 | }, 100 | }, 101 | ) 102 | 103 | // we can spawn worker for each interface, but i doubt that we will ever need such 104 | // optimization 105 | go c.worker() 106 | go controller.Run(stopCh) 107 | <-stopCh 108 | c.Queue.Close() 109 | } 110 | 111 | func (c *ExternalIpController) worker() { 112 | for { 113 | item, quit := c.Queue.Get() 114 | if quit { 115 | return 116 | } 117 | c.processItem(item) 118 | } 119 | } 120 | 121 | func (c *ExternalIpController) processItem(item interface{}) { 122 | defer c.Queue.Done(item) 123 | var err error 124 | var cidr string 125 | var action string 126 | switch t := item.(type) { 127 | case *netutils.AddCIDR: 128 | err = c.ipHandler.Add(c.Iface, t.Cidr) 129 | cidr = t.Cidr 130 | action = "assignment" 131 | 132 | case *netutils.DelCIDR: 133 | err = c.ipHandler.Del(c.Iface, t.Cidr) 134 | cidr = t.Cidr 135 | action = "removal" 136 | } 137 | if err != nil { 138 | glog.Errorf("Error during %s of IP %v on %s - %v", action, cidr, c.Iface, err) 139 | c.Queue.Add(item) 140 | } else { 141 | glog.V(2).Infof("%s of IP %v was done successfully", action, cidr) 142 | } 143 | } 144 | 145 | func boolMapDifference(minuend, subtrahend map[string]bool) map[string]bool { 146 | difference := make(map[string]bool) 147 | 148 | for key := range minuend { 149 | if !subtrahend[key] { 150 | difference[key] = true 151 | } 152 | } 153 | 154 | return difference 155 | } 156 | 157 | func neglectIPsInUse(ips map[string]bool, key string, store cache.Store) { 158 | svcList := store.List() 159 | for s := range svcList { 160 | svc := svcList[s].(*v1.Service) 161 | svcKey, _ := cache.MetaNamespaceKeyFunc(svc) 162 | if svcKey != key { 163 | for _, ip := range svc.Spec.ExternalIPs { 164 | delete(ips, ip) 165 | } 166 | } 167 | } 168 | } 169 | 170 | func (c *ExternalIpController) processServiceExternalIPs(old, cur *v1.Service, store cache.Store) { 171 | old_ips := make(map[string]bool) 172 | cur_ips := make(map[string]bool) 173 | key := "" 174 | 175 | if old != nil { 176 | for i := range old.Spec.ExternalIPs { 177 | old_ips[old.Spec.ExternalIPs[i]] = true 178 | } 179 | key, _ = cache.MetaNamespaceKeyFunc(old) 180 | } 181 | if cur != nil { 182 | for i := range cur.Spec.ExternalIPs { 183 | cur_ips[cur.Spec.ExternalIPs[i]] = true 184 | } 185 | key, _ = cache.MetaNamespaceKeyFunc(cur) 186 | } 187 | 188 | if reflect.DeepEqual(cur_ips, old_ips) { 189 | return 190 | } 191 | 192 | ips_to_add := boolMapDifference(cur_ips, old_ips) 193 | ips_to_remove := boolMapDifference(old_ips, cur_ips) 194 | 195 | neglectIPsInUse(ips_to_add, key, store) 196 | neglectIPsInUse(ips_to_remove, key, store) 197 | 198 | for ip := range ips_to_add { 199 | cidr := ip + "/" + c.Mask 200 | c.Queue.Add(&netutils.AddCIDR{cidr}) 201 | } 202 | for ip := range ips_to_remove { 203 | cidr := ip + "/" + c.Mask 204 | c.Queue.Add(&netutils.DelCIDR{cidr}) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /pkg/claimcontroller/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package claimcontroller 16 | 17 | import ( 18 | "strings" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 23 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 24 | 25 | "github.com/golang/glog" 26 | "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | "k8s.io/apimachinery/pkg/watch" 30 | "k8s.io/client-go/kubernetes" 31 | "k8s.io/client-go/rest" 32 | "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | func NewClaimController(iface, uid string, config *rest.Config, resyncInterval time.Duration, hbInterval time.Duration) (*claimController, error) { 36 | clientset, err := kubernetes.NewForConfig(config) 37 | if err != nil { 38 | return nil, err 39 | } 40 | ext, err := extensions.WrapClientsetWithExtensions(clientset, config) 41 | if err != nil { 42 | return nil, err 43 | } 44 | claimSource := &cache.ListWatch{ 45 | ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 46 | return ext.IPClaims().List(options) 47 | }, 48 | WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 49 | glog.V(3).Infof("Calling claim watcher with options %v", options) 50 | return ext.IPClaims().Watch(options) 51 | }, 52 | } 53 | queue := workqueue.NewQueue() 54 | uid = strings.Replace(uid, ".", "-", -1) 55 | return &claimController{ 56 | Clientset: clientset, 57 | ExtensionsClientset: ext, 58 | Iface: iface, 59 | Uid: uid, 60 | claimSource: claimSource, 61 | queue: queue, 62 | iphandler: netutils.LinuxIPHandler{}, 63 | heartbeatPeriod: hbInterval, 64 | resyncInterval: resyncInterval, 65 | }, nil 66 | } 67 | 68 | type claimController struct { 69 | Clientset *kubernetes.Clientset 70 | ExtensionsClientset extensions.ExtensionsClientset 71 | // i am not sure that it should be configurable for controller 72 | Iface string 73 | Uid string 74 | 75 | claimSource cache.ListerWatcher 76 | claimStore cache.Store 77 | 78 | queue workqueue.QueueType 79 | iphandler netutils.IPHandler 80 | 81 | // heartbeatPeriod for a node, should be < monitorPeriod in scheduller 82 | heartbeatPeriod time.Duration 83 | 84 | resyncInterval time.Duration 85 | } 86 | 87 | func (c *claimController) Run(stop chan struct{}) { 88 | go c.worker() 89 | go c.claimWatcher(stop) 90 | go c.heartbeatIpNode(stop, time.Tick(c.heartbeatPeriod)) 91 | <-stop 92 | c.queue.Close() 93 | } 94 | 95 | func (c *claimController) claimWatcher(stop chan struct{}) { 96 | store, controller := cache.NewInformer( 97 | c.claimSource, 98 | &extensions.IpClaim{}, 99 | c.resyncInterval, 100 | cache.ResourceEventHandlerFuncs{ 101 | AddFunc: func(obj interface{}) { 102 | claim := obj.(*extensions.IpClaim) 103 | glog.V(3).Infof("Received add event for ipclaim %v - %v - %v", 104 | claim.Metadata.Name, claim.Spec.NodeName, claim.Metadata.ResourceVersion) 105 | c.queue.Add(claim) 106 | }, 107 | UpdateFunc: func(old, cur interface{}) { 108 | oldClaim := old.(*extensions.IpClaim) 109 | curClaim := cur.(*extensions.IpClaim) 110 | glog.V(3).Infof("Received update event. Old ipclaim %v - %v, New ipclaim %v - %v -%v", 111 | oldClaim.Metadata.Name, oldClaim.Spec.NodeName, 112 | curClaim.Metadata.Name, curClaim.Spec.NodeName, curClaim.Metadata.ResourceVersion) 113 | c.queue.Add(curClaim) 114 | }, 115 | DeleteFunc: func(obj interface{}) { 116 | claim := obj.(*extensions.IpClaim) 117 | glog.V(3).Infof("Received delete event for %v - %v. Resource version %v", 118 | claim.Metadata.Name, claim.Spec.NodeName, claim.Metadata.ResourceVersion) 119 | c.queue.Add(claim) 120 | }, 121 | }, 122 | ) 123 | c.claimStore = store 124 | controller.Run(stop) 125 | } 126 | 127 | func (c *claimController) worker() { 128 | for { 129 | item, quit := c.queue.Get() 130 | if quit { 131 | return 132 | } 133 | err := c.processClaim(item.(*extensions.IpClaim)) 134 | if err != nil { 135 | glog.Errorf("Error processing claim %v", err) 136 | c.queue.Add(item) 137 | } 138 | c.queue.Done(item) 139 | } 140 | } 141 | 142 | func (c *claimController) processClaim(ipclaim *extensions.IpClaim) error { 143 | glog.V(5).Infof("Processing claim %v with node %v and uid %v", 144 | ipclaim.Spec.Cidr, ipclaim.Spec.NodeName, c.Uid) 145 | if _, exists, _ := c.claimStore.Get(ipclaim); !exists { 146 | return c.iphandler.Del(c.Iface, ipclaim.Spec.Cidr) 147 | } 148 | if ipclaim.Spec.NodeName == c.Uid { 149 | return c.iphandler.Add(c.Iface, ipclaim.Spec.Cidr) 150 | } else { 151 | return c.iphandler.Del(c.Iface, ipclaim.Spec.Cidr) 152 | } 153 | } 154 | 155 | func (c *claimController) heartbeatIpNode(stop chan struct{}, ticker <-chan time.Time) { 156 | for { 157 | select { 158 | case <-stop: 159 | return 160 | case <-ticker: 161 | ipnode, err := c.ExtensionsClientset.IPNodes().Get(c.Uid) 162 | if errors.IsNotFound(err) { 163 | ipnode := &extensions.IpNode{ 164 | Metadata: metav1.ObjectMeta{Name: c.Uid}, 165 | } 166 | _, err := c.ExtensionsClientset.IPNodes().Create(ipnode) 167 | if err != nil { 168 | glog.Errorf("Error creating node %v : %v", c.Uid, err) 169 | } 170 | continue 171 | } 172 | 173 | if err != nil { 174 | glog.Errorf("Error fetching node %v : %v", c.Uid, err) 175 | continue 176 | } 177 | glog.V(3).Infof("Updating ipnode %v. Version %v.", 178 | ipnode.Metadata.Name, ipnode.Revision) 179 | ipnode.Revision++ 180 | _, err = c.ExtensionsClientset.IPNodes().Update(ipnode) 181 | if err != nil { 182 | glog.Errorf("Error updating node %v : %v", c.Uid, err) 183 | continue 184 | } 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /pkg/extensions/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "encoding/json" 19 | "errors" 20 | "net" 21 | 22 | "k8s.io/apimachinery/pkg/apimachinery/announced" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/apimachinery/pkg/runtime/schema" 26 | "k8s.io/client-go/pkg/api" 27 | 28 | "github.com/Mirantis/k8s-externalipcontroller/pkg/netutils" 29 | ) 30 | 31 | const ( 32 | GroupName string = "ipcontroller.ext" 33 | Version string = "v1" 34 | ) 35 | 36 | var ( 37 | SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: Version} 38 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 39 | ) 40 | 41 | func addKnownTypes(scheme *runtime.Scheme) error { 42 | scheme.AddKnownTypes( 43 | SchemeGroupVersion, 44 | &IpNode{}, 45 | &IpNodeList{}, 46 | &IpClaim{}, 47 | &IpClaimList{}, 48 | &IpClaimPool{}, 49 | &IpClaimPoolList{}, 50 | 51 | &metav1.GetOptions{}, 52 | &metav1.ListOptions{}, 53 | &metav1.DeleteOptions{}, 54 | ) 55 | return nil 56 | } 57 | 58 | func init() { 59 | if err := announced.NewGroupMetaFactory( 60 | &announced.GroupMetaFactoryArgs{ 61 | GroupName: GroupName, 62 | VersionPreferenceOrder: []string{SchemeGroupVersion.Version}, 63 | AddInternalObjectsToScheme: SchemeBuilder.AddToScheme, 64 | }, 65 | announced.VersionToSchemeFunc{ 66 | SchemeGroupVersion.Version: SchemeBuilder.AddToScheme, 67 | }, 68 | ).Announce(api.GroupFactoryRegistry).RegisterAndEnable(api.Registry, api.Scheme); err != nil { 69 | panic(err) 70 | } 71 | } 72 | 73 | type IpNode struct { 74 | metav1.TypeMeta `json:",inline"` 75 | 76 | // Standard object metadata 77 | Metadata metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 78 | 79 | // used as a heartbeat 80 | Revision int64 `json:",string"` 81 | } 82 | 83 | func (e *IpNode) GetObjectKind() schema.ObjectKind { 84 | return &e.TypeMeta 85 | } 86 | 87 | func (e *IpNode) GetObjectMeta() metav1.Object { 88 | return &e.Metadata 89 | } 90 | 91 | type IpNodeList struct { 92 | metav1.TypeMeta `json:",inline"` 93 | 94 | // Standard list metadata. 95 | metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 96 | 97 | Items []IpNode `json:"items" protobuf:"bytes,2,rep,name=items"` 98 | } 99 | 100 | type IpClaim struct { 101 | metav1.TypeMeta `json:",inline"` 102 | 103 | // Standard object metadata 104 | Metadata metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 105 | 106 | Spec IpClaimSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` 107 | } 108 | 109 | func (e *IpClaim) GetObjectKind() schema.ObjectKind { 110 | return &e.TypeMeta 111 | } 112 | 113 | func (e *IpClaim) GetObjectMeta() metav1.Object { 114 | return &e.Metadata 115 | } 116 | 117 | type IpClaimList struct { 118 | metav1.TypeMeta `json:",inline"` 119 | 120 | // Standard list metadata. 121 | metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 122 | 123 | Items []IpClaim `json:"items" protobuf:"bytes,2,rep,name=items"` 124 | } 125 | 126 | type IpClaimSpec struct { 127 | // NodeName used to identify where IPClaim is assigned (IPNode.Name) 128 | NodeName string `json:"nodeName" protobuf:"bytes,10,opt,name=nodeName"` 129 | Cidr string `json:"cidr,omitempty" protobuf:"bytes,10,opt,name=cidr"` 130 | Link string `json:"link" protobuf:"bytes,10,opt,name=link"` 131 | } 132 | 133 | type IpClaimPool struct { 134 | metav1.TypeMeta `json:",inline"` 135 | 136 | // Standard object metadata 137 | Metadata metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 138 | 139 | Spec IpClaimPoolSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` 140 | } 141 | 142 | type IpClaimPoolSpec struct { 143 | CIDR string `json:"cidr" protobuf:"bytes,10,opt,name=cidr"` 144 | Ranges [][]string `json:"ranges,omitempty" protobuf:"bytes,5,opt,name=ranges"` 145 | Allocated map[string]string `json:"allocated,omitempty" protobuf:"bytes,2,opt,name=allocated"` 146 | } 147 | 148 | type IpClaimPoolList struct { 149 | metav1.TypeMeta `json:",inline"` 150 | 151 | // Standard list metadata. 152 | metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 153 | 154 | Items []IpClaimPool `json:"items" protobuf:"bytes,2,rep,name=items"` 155 | } 156 | 157 | func (p *IpClaimPool) AvailableIP() (availableIP string, err error) { 158 | ip, network, err := net.ParseCIDR(p.Spec.CIDR) 159 | if err != nil { 160 | return 161 | } 162 | 163 | var dropOffIP net.IP 164 | 165 | ranges := [][]net.IP{} 166 | 167 | //in case 'Ranges' is not set for the pool assume ranges of the 168 | //network itself 169 | if p.Spec.Ranges != nil { 170 | for _, r := range p.Spec.Ranges { 171 | ip = net.ParseIP(r[0]) 172 | dropOffIP = net.ParseIP(r[len(r)-1]) 173 | netutils.IPIncrement(dropOffIP) 174 | 175 | ranges = append(ranges, []net.IP{ip, dropOffIP}) 176 | } 177 | } else { 178 | //network address is not usable 179 | netutils.IPIncrement(ip) 180 | ranges = [][]net.IP{{ip, dropOffIP}} 181 | } 182 | 183 | for _, r := range ranges { 184 | curAddr := r[0] 185 | nextAddr := make(net.IP, len(curAddr)) 186 | copy(nextAddr, curAddr) 187 | netutils.IPIncrement(nextAddr) 188 | 189 | firstOut := r[len(r)-1] 190 | 191 | for network.Contains(curAddr) && network.Contains(nextAddr) && !curAddr.Equal(firstOut) { 192 | if _, exists := p.Spec.Allocated[curAddr.String()]; !exists { 193 | return curAddr.String(), nil 194 | } 195 | netutils.IPIncrement(curAddr) 196 | netutils.IPIncrement(nextAddr) 197 | 198 | } 199 | } 200 | 201 | return "", errors.New("There is no free IP left in the pool") 202 | } 203 | 204 | func (p *IpClaimPool) GetObjectKind() schema.ObjectKind { 205 | return &p.TypeMeta 206 | } 207 | 208 | func (p *IpClaimPool) GetObjectMeta() metav1.Object { 209 | return &p.Metadata 210 | } 211 | 212 | // see https://github.com/kubernetes/client-go/issues/8 213 | type ExampleIpNode IpNode 214 | type ExampleIpNodesList IpNodeList 215 | type ExampleIpClaim IpClaim 216 | type ExampleIpClaimList IpClaimList 217 | type ExampleIpClaimPool IpClaimPool 218 | type ExampleIpClaimPoolList IpClaimPoolList 219 | 220 | func (e *IpClaimPool) UnmarshalJSON(data []byte) error { 221 | tmp := ExampleIpClaimPool{} 222 | err := json.Unmarshal(data, &tmp) 223 | if err != nil { 224 | return err 225 | } 226 | tmp2 := IpClaimPool(tmp) 227 | *e = tmp2 228 | return nil 229 | } 230 | 231 | func (e *IpClaimPoolList) UnmarshalJSON(data []byte) error { 232 | tmp := ExampleIpClaimPoolList{} 233 | err := json.Unmarshal(data, &tmp) 234 | if err != nil { 235 | return err 236 | } 237 | tmp2 := IpClaimPoolList(tmp) 238 | *e = tmp2 239 | return nil 240 | } 241 | 242 | func (e *IpNode) UnmarshalJSON(data []byte) error { 243 | tmp := ExampleIpNode{} 244 | err := json.Unmarshal(data, &tmp) 245 | if err != nil { 246 | return err 247 | } 248 | tmp2 := IpNode(tmp) 249 | *e = tmp2 250 | return nil 251 | } 252 | 253 | func (el *IpNodeList) UnmarshalJSON(data []byte) error { 254 | tmp := ExampleIpNodesList{} 255 | err := json.Unmarshal(data, &tmp) 256 | if err != nil { 257 | return err 258 | } 259 | tmp2 := IpNodeList(tmp) 260 | *el = tmp2 261 | return nil 262 | } 263 | 264 | func (e *IpClaim) UnmarshalJSON(data []byte) error { 265 | tmp := ExampleIpClaim{} 266 | err := json.Unmarshal(data, &tmp) 267 | if err != nil { 268 | return err 269 | } 270 | tmp2 := IpClaim(tmp) 271 | *e = tmp2 272 | return nil 273 | } 274 | 275 | func (el *IpClaimList) UnmarshalJSON(data []byte) error { 276 | tmp := ExampleIpClaimList{} 277 | err := json.Unmarshal(data, &tmp) 278 | if err != nil { 279 | return err 280 | } 281 | tmp2 := IpClaimList(tmp) 282 | *el = tmp2 283 | return nil 284 | } 285 | -------------------------------------------------------------------------------- /doc/ecmp-load-balancing.md: -------------------------------------------------------------------------------- 1 | ECMP Load Balancing for External IPs 2 | ==================================== 3 | 4 | - [ECMP Load Balancing for External IPs](#ecmp-load-balancing-for-external-ips) 5 | * [Introduction](#introduction) 6 | * [Proposed solution](#proposed-solution) 7 | * [Deployment scheme](#deployment-scheme) 8 | + [Diagram](#diagram) 9 | + [Description](#description) 10 | * [Single rack network topology](#single-rack-network-topology) 11 | + [Diagram](#diagram-1) 12 | + [Description](#description-1) 13 | + [Example](#example) 14 | * [Multirack network topology](#multirack-network-topology) 15 | + [Diagram](#diagram-2) 16 | + [Description](#description-2) 17 | + [Example](#example-1) 18 | 19 | ## Introduction 20 | 21 | One of the possible ways to expose k8s services on a bare metal deployments is 22 | using External IPs. Each node runs a kube-proxy process which programs `iptables` 23 | rules to trap access to External IPs and redirect them to the correct backends. 24 | 25 | So in order to access k8s service from the outside we just need to route public 26 | traffic to one of k8s worker nodes which has `kube-proxy` running and thus has 27 | needed `iptables` rules for External IPs. 28 | 29 | ## Proposed solution 30 | 31 | External IP controller is k8s application which is deployed on top of k8s 32 | cluster and which configures External IPs on k8s worker node(s) to provide 33 | IP connectivity. 34 | 35 | This document describes how to expose Kubernetes services using External IP 36 | controller and how to provide load balancing and high availability for External 37 | IPs based on Equal-cost multi-path routing (ECMP). Proposed network topologies 38 | should be considered as examples. 39 | 40 | ## Deployment scheme 41 | 42 | ### Diagram 43 | 44 | ![Deployment scheme](images/ecmp-scheme.png) 45 | 46 | ### Description 47 | 48 | * External IP controller kubernetes application is running on all nodes (or on 49 | a subet of nodes) as a DeamonSet or ReplicaSet with node affinity (up to cloud 50 | operator). 51 | * On start every External IP controller instance pulls information about 52 | services from `kube-api` and brings up all External IPs on `lo` interface on 53 | all target nodes. 54 | * Every External IP controller instance watches `kube-api` for updates in 55 | services with External IPs and: 56 | * When new External IPs appear every instance brings them up on `lo`. 57 | * When service is removed every instance removes appropriate External IPs 58 | from the interface. 59 | * External IP BGP speaker k8s application is running on the same nodes as 60 | External IP controller. It monitors local system and announces External IPs 61 | found on `lo` via BGP. 62 | 63 | ## Single rack network topology 64 | 65 | ### Diagram 66 | 67 | ![Simple network topology](images/bgp-for-ecmp-single.png) 68 | 69 | ### Description 70 | 71 | * There's one instance of Route-Reflector application running in the AS. This 72 | application (k8s PoD) should have static IP which should migrate with it to 73 | the new node in case of fail-over. Otherwise Top of Rack router configuration 74 | should be updated when Route-Reflector IP changes. 75 | * ExIP BGP speakers and Route-Reflector are configured as neighbors within 76 | the same AS. 77 | * Route-Reflector is peered with Top of Rack router within the same AS. 78 | * External IPs assigned by External IP controller on `lo` are exported by 79 | ExIP BGP speakers to the Route-Reflector and thus to Top of Rack router. 80 | * Top of Rack router should have ECMP enabled/configured (for virtual lab and 81 | BIRD routing daemon `merge paths` should be enabled in `kernel` protocol to 82 | support ECMP in routing table). 83 | 84 | ### Example 85 | 86 | Let's see how it works on example: 87 | * Kuberenetes cluster is running on 3 nodes: 88 | ``` 89 | node1 10.210.1.11 90 | node2 10.210.1.12 91 | node3 10.210.1.13 92 | ``` 93 | * External IP controller is running on 2 nodes (node1 and node3), 94 | [example](../examples/simple/externalipcontroller.yaml) with `replicas: 2` and 95 | `HOST_INTERFACE` env variable set to `lo`. 96 | * User creates a service with external IPs ([example](../examples/nginx.yaml)): 97 | ``` 98 | externalIPs: 99 | - 10.0.0.7 100 | - 10.0.0.8 101 | ``` 102 | * External IP controller assigns those IPs to `lo` interface of node1 and 103 | node3: 104 | ``` 105 | 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1 106 | inet 127.0.0.1/8 scope host lo 107 | valid_lft forever preferred_lft forever 108 | inet 10.0.0.8/32 scope global lo 109 | valid_lft forever preferred_lft forever 110 | inet 10.0.0.7/32 scope global lo 111 | valid_lft forever preferred_lft forever 112 | ``` 113 | * ExIP BGP speakers (containers with BIRD) detect changes on node1/node3 and 114 | announce them to peers via BGP. Example of BIRD export filter: 115 | ``` 116 | if ( ifname = "lo" ) then { 117 | if net != 127.0.0.0/8 then accept; 118 | } 119 | ``` 120 | * Top of Rack router (BIRD daemon with `merge paths` enabled) configures the 121 | following routes: 122 | ``` 123 | 10.0.0.7 proto bird 124 | nexthop via 10.210.1.11 dev eth2 weight 1 125 | nexthop via 10.210.1.13 dev eth2 weight 1 126 | 10.0.0.8 proto bird 127 | nexthop via 10.210.1.11 dev eth2 weight 1 128 | nexthop via 10.210.1.13 dev eth2 weight 1 129 | ``` 130 | 131 | ## Multirack network topology 132 | 133 | ### Diagram 134 | 135 | ![Multirack network topology](images/bgp-for-ecmp.png) 136 | 137 | ### Description 138 | 139 | * There's one instance of Route-Reflector application running per rack/AS. This 140 | application (k8s PoD) should have static IP which should migrate with it to 141 | the new node in case of fail-over. Otherwise Top of Rack router configuration 142 | should be updated when Route-Reflector IP changes. 143 | * ExIP BGP speakers and Route-Reflector are configured as neighbors within the 144 | same AS. 145 | See [ExIP BGP speaker configuration example](examples/bird-node1.cfg). 146 | * Route-Reflector is peered with Top of Rack router within the same AS. ECMP 147 | should be configured on both sides (for virtual lab and BIRD routing daemon 148 | `add paths` should be enabled in BGP protocol). 149 | See [Route-Reflector configuration example](examples/bird-rr1.cfg). 150 | * External IPs assigned by External IP controller on `lo` are exported by 151 | ExIP BGP speakers to the Route-Reflector and thus to Top of Rack router. 152 | * Top of Rack router should have ECMP enabled/configured (for virtual lab and 153 | BIRD routing daemon `merge paths` should be enabled in `kernel` protocol to 154 | support ECMP in routing table). 155 | See [Top of Rack router BGP configuration example](examples/bird-tor1.cfg). 156 | * Core router is peered with Top of Rack routers via eBGP. It also has ECMP 157 | enabled/configured (for virtual lab and BIRD routing daemon `merge paths` 158 | should be enabled in `kernel` protocol to support ECMP in routing table). 159 | See [Core router BGP configuration example](examples/bird-core.cfg). 160 | 161 | ### Example 162 | 163 | ![Multirack example topology](images/bgp-for-ecmp-example.png) 164 | 165 | * `nginx` pod and service are running ([example](../examples/nginx.yaml)): 166 | ``` 167 | default nginx-3086523004-0aqgr 1/1 Running 0 1h 10.233.80.3 node-01-003 168 | default nginx-3086523004-3t6hq 1/1 Running 0 2m 10.233.79.67 node-02-002 169 | default nginx-3086523004-ca67h 1/1 Running 0 1h 10.233.85.4 node-01-002 170 | ``` 171 | * Let's check route from `node-01-002` to nginx POD running in rack2: 172 | ``` 173 | $ traceroute -n 10.233.79.67 174 | traceroute to 10.233.79.67 (10.233.79.67), 30 hops max, 60 byte packets 175 | 1 10.211.1.254 0.352 ms 0.316 ms 0.247 ms 176 | 2 192.168.192.5 0.301 ms 0.409 ms 0.353 ms 177 | 3 192.168.192.10 0.459 ms 0.392 ms 0.703 ms 178 | 4 10.211.2.2 0.933 ms 0.879 ms 1.166 ms 179 | 5 10.233.79.67 1.453 ms 1.396 ms 1.246 ms 180 | ``` 181 | * `nginx` service has 2 external IPs (10.0.0.7 and 10.0.0.8). External IP controller 182 | assigned those IPs to `lo` interface on all nodes: 183 | ``` 184 | 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1 185 | inet 127.0.0.1/8 scope host lo 186 | valid_lft forever preferred_lft forever 187 | inet 10.0.0.8/32 scope global lo 188 | valid_lft forever preferred_lft forever 189 | inet 10.0.0.7/32 scope global lo 190 | valid_lft forever preferred_lft forever 191 | ``` 192 | * Routing tables on the routers: 193 | 194 | Core router 195 | ``` 196 | 10.0.0.7 proto bird 197 | nexthop via 192.168.192.6 dev rack01a weight 1 198 | nexthop via 192.168.192.10 dev rack02a weight 1 199 | 10.0.0.8 proto bird 200 | nexthop via 192.168.192.6 dev rack01a weight 1 201 | nexthop via 192.168.192.10 dev rack02a weight 1 202 | ``` 203 | 204 | TOR1 205 | ``` 206 | 10.0.0.7 proto bird 207 | nexthop via 10.211.1.2 dev eth2 weight 1 208 | nexthop via 10.211.1.3 dev eth2 weight 1 209 | 10.0.0.8 proto bird 210 | nexthop via 10.211.1.2 dev eth2 weight 1 211 | nexthop via 10.211.1.3 dev eth2 weight 1 212 | ``` 213 | 214 | TOR2 (no multipath since we have only one node in rack2) 215 | ``` 216 | 10.0.0.7 via 10.211.2.2 dev eth3 proto bird 217 | 10.0.0.8 via 10.211.2.2 dev eth3 proto bird 218 | ``` 219 | -------------------------------------------------------------------------------- /pkg/extensions/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package extensions 16 | 17 | import ( 18 | "bytes" 19 | "encoding/json" 20 | 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/labels" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | "k8s.io/apimachinery/pkg/runtime/schema" 25 | "k8s.io/apimachinery/pkg/runtime/serializer" 26 | "k8s.io/apimachinery/pkg/watch" 27 | "k8s.io/client-go/kubernetes" 28 | "k8s.io/client-go/pkg/api" 29 | "k8s.io/client-go/rest" 30 | ) 31 | 32 | func WrapClientsetWithExtensions(clientset *kubernetes.Clientset, config *rest.Config) (*WrappedClientset, error) { 33 | restConfig := &rest.Config{} 34 | *restConfig = *config 35 | rest, err := extensionClient(restConfig) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return &WrappedClientset{ 40 | Client: rest, 41 | }, nil 42 | } 43 | 44 | func extensionClient(config *rest.Config) (*rest.RESTClient, error) { 45 | config.APIPath = "/apis" 46 | config.ContentConfig = rest.ContentConfig{ 47 | GroupVersion: &schema.GroupVersion{ 48 | Group: GroupName, 49 | Version: Version, 50 | }, 51 | NegotiatedSerializer: serializer.DirectCodecFactory{CodecFactory: api.Codecs}, 52 | ContentType: runtime.ContentTypeJSON, 53 | } 54 | return rest.RESTClientFor(config) 55 | } 56 | 57 | type ExtensionsClientset interface { 58 | IPNodes() IPNodesInterface 59 | IPClaims() IPClaimsInterface 60 | IPClaimPools() IPClaimPoolsInterface 61 | } 62 | 63 | type WrappedClientset struct { 64 | Client *rest.RESTClient 65 | } 66 | 67 | type IPClaimsInterface interface { 68 | Create(*IpClaim) (*IpClaim, error) 69 | Get(name string) (*IpClaim, error) 70 | List(metav1.ListOptions) (*IpClaimList, error) 71 | Watch(metav1.ListOptions) (watch.Interface, error) 72 | Update(*IpClaim) (*IpClaim, error) 73 | Delete(string, *metav1.DeleteOptions) error 74 | } 75 | 76 | type IPNodesInterface interface { 77 | Create(*IpNode) (*IpNode, error) 78 | Get(name string) (*IpNode, error) 79 | List(metav1.ListOptions) (*IpNodeList, error) 80 | Watch(metav1.ListOptions) (watch.Interface, error) 81 | Update(*IpNode) (*IpNode, error) 82 | Delete(string, *metav1.DeleteOptions) error 83 | } 84 | 85 | type IPClaimPoolsInterface interface { 86 | Create(*IpClaimPool) (*IpClaimPool, error) 87 | Get(name string) (*IpClaimPool, error) 88 | List(metav1.ListOptions) (*IpClaimPoolList, error) 89 | Update(*IpClaimPool) (*IpClaimPool, error) 90 | Delete(string, *metav1.DeleteOptions) error 91 | } 92 | 93 | func (w *WrappedClientset) IPNodes() IPNodesInterface { 94 | return &IPNodesClient{w.Client} 95 | } 96 | 97 | func (w *WrappedClientset) IPClaims() IPClaimsInterface { 98 | return &IpClaimClient{w.Client} 99 | } 100 | 101 | func (w *WrappedClientset) IPClaimPools() IPClaimPoolsInterface { 102 | return &IpClaimPoolClient{w.Client} 103 | } 104 | 105 | type IPNodesClient struct { 106 | client *rest.RESTClient 107 | } 108 | 109 | type IpClaimClient struct { 110 | client *rest.RESTClient 111 | } 112 | 113 | type IpClaimPoolClient struct { 114 | client *rest.RESTClient 115 | } 116 | 117 | func decodeResponseInto(resp []byte, obj interface{}) error { 118 | return json.NewDecoder(bytes.NewReader(resp)).Decode(obj) 119 | } 120 | 121 | func (c *IPNodesClient) Create(ipnode *IpNode) (result *IpNode, err error) { 122 | result = &IpNode{} 123 | resp, err := c.client.Post(). 124 | Namespace("default"). 125 | Resource("ipnodes"). 126 | Body(ipnode). 127 | DoRaw() 128 | if err != nil { 129 | return result, err 130 | } 131 | return result, decodeResponseInto(resp, result) 132 | } 133 | 134 | func (c *IPNodesClient) List(opts metav1.ListOptions) (result *IpNodeList, err error) { 135 | result = &IpNodeList{} 136 | selector, err := labels.Parse(opts.LabelSelector) 137 | if err != nil { 138 | return nil, err 139 | } 140 | resp, err := c.client.Get(). 141 | Namespace("default"). 142 | Resource("ipnodes"). 143 | LabelsSelectorParam(selector). 144 | DoRaw() 145 | if err != nil { 146 | return result, err 147 | } 148 | return result, decodeResponseInto(resp, result) 149 | } 150 | 151 | func (c *IPNodesClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { 152 | return c.client.Get(). 153 | Namespace("default"). 154 | Prefix("watch"). 155 | Resource("ipnodes"). 156 | VersionedParams(&opts, api.ParameterCodec). 157 | Watch() 158 | } 159 | 160 | func (c *IPNodesClient) Update(ipnode *IpNode) (result *IpNode, err error) { 161 | result = &IpNode{} 162 | resp, err := c.client.Put(). 163 | Namespace("default"). 164 | Resource("ipnodes"). 165 | Name(ipnode.Metadata.Name). 166 | Body(ipnode). 167 | DoRaw() 168 | if err != nil { 169 | return result, err 170 | } 171 | return result, decodeResponseInto(resp, result) 172 | } 173 | 174 | func (c *IPNodesClient) Delete(name string, options *metav1.DeleteOptions) error { 175 | return c.client.Delete(). 176 | Namespace("default"). 177 | Resource("ipnodes"). 178 | Name(name). 179 | Body(options). 180 | Do(). 181 | Error() 182 | } 183 | 184 | func (c *IPNodesClient) Get(name string) (result *IpNode, err error) { 185 | result = &IpNode{} 186 | resp, err := c.client.Get(). 187 | Namespace("default"). 188 | Resource("ipnodes"). 189 | Name(name). 190 | DoRaw() 191 | if err != nil { 192 | return result, err 193 | } 194 | return result, decodeResponseInto(resp, result) 195 | } 196 | 197 | func (c *IpClaimClient) Get(name string) (result *IpClaim, err error) { 198 | result = &IpClaim{} 199 | err = c.client.Get(). 200 | Namespace("default"). 201 | Resource("ipclaims"). 202 | Name(name). 203 | Do(). 204 | Into(result) 205 | 206 | return result, err 207 | } 208 | 209 | func (c *IpClaimClient) Create(ipclaim *IpClaim) (result *IpClaim, err error) { 210 | result = &IpClaim{} 211 | resp, err := c.client.Post(). 212 | Namespace("default"). 213 | Resource("ipclaims"). 214 | Body(ipclaim). 215 | DoRaw() 216 | if err != nil { 217 | return result, err 218 | } 219 | return result, decodeResponseInto(resp, result) 220 | } 221 | 222 | func (c *IpClaimClient) List(opts metav1.ListOptions) (result *IpClaimList, err error) { 223 | result = &IpClaimList{} 224 | selector, err := labels.Parse(opts.LabelSelector) 225 | if err != nil { 226 | return nil, err 227 | } 228 | resp, err := c.client.Get(). 229 | Namespace("default"). 230 | Resource("ipclaims"). 231 | LabelsSelectorParam(selector). 232 | DoRaw() 233 | if err != nil { 234 | return result, err 235 | } 236 | return result, decodeResponseInto(resp, result) 237 | } 238 | 239 | func (c *IpClaimClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { 240 | return c.client.Get(). 241 | Namespace("default"). 242 | Prefix("watch"). 243 | Resource("ipclaims"). 244 | Param("resourceVersion", opts.ResourceVersion). 245 | Watch() 246 | } 247 | 248 | func (c *IpClaimClient) Update(ipclaim *IpClaim) (result *IpClaim, err error) { 249 | result = &IpClaim{} 250 | resp, err := c.client.Put(). 251 | Namespace("default"). 252 | Resource("ipclaims"). 253 | Name(ipclaim.Metadata.Name). 254 | Body(ipclaim). 255 | DoRaw() 256 | if err != nil { 257 | return result, err 258 | } 259 | return result, decodeResponseInto(resp, result) 260 | } 261 | 262 | func (c *IpClaimClient) Delete(name string, options *metav1.DeleteOptions) error { 263 | return c.client.Delete(). 264 | Namespace("default"). 265 | Resource("ipclaims"). 266 | Name(name). 267 | Body(options). 268 | Do(). 269 | Error() 270 | } 271 | 272 | func (c *IpClaimPoolClient) Get(name string) (result *IpClaimPool, err error) { 273 | result = &IpClaimPool{} 274 | err = c.client.Get(). 275 | Namespace("default"). 276 | Resource("ipclaimpools"). 277 | Name(name). 278 | Do(). 279 | Into(result) 280 | 281 | return result, err 282 | } 283 | 284 | func (c *IpClaimPoolClient) Create(ipclaimpool *IpClaimPool) (result *IpClaimPool, err error) { 285 | result = &IpClaimPool{} 286 | resp, err := c.client.Post(). 287 | Namespace("default"). 288 | Resource("ipclaimpools"). 289 | Body(ipclaimpool). 290 | DoRaw() 291 | if err != nil { 292 | return result, err 293 | } 294 | return result, decodeResponseInto(resp, result) 295 | } 296 | 297 | func (c *IpClaimPoolClient) List(opts metav1.ListOptions) (result *IpClaimPoolList, err error) { 298 | result = &IpClaimPoolList{} 299 | resp, err := c.client.Get(). 300 | Namespace("default"). 301 | Resource("ipclaimpools"). 302 | VersionedParams(&opts, api.ParameterCodec). 303 | DoRaw() 304 | if err != nil { 305 | return result, err 306 | } 307 | return result, decodeResponseInto(resp, result) 308 | } 309 | 310 | func (c *IpClaimPoolClient) Delete(name string, options *metav1.DeleteOptions) error { 311 | return c.client.Delete(). 312 | Namespace("default"). 313 | Resource("ipclaimpools"). 314 | Name(name). 315 | Body(options). 316 | Do(). 317 | Error() 318 | } 319 | 320 | func (c *IpClaimPoolClient) Update(ipclaimpool *IpClaimPool) (result *IpClaimPool, err error) { 321 | result = &IpClaimPool{} 322 | resp, err := c.client.Put(). 323 | Namespace("default"). 324 | Resource("ipclaimpools"). 325 | Name(ipclaimpool.Metadata.Name). 326 | Body(ipclaimpool). 327 | DoRaw() 328 | if err != nil { 329 | return result, err 330 | } 331 | return result, decodeResponseInto(resp, result) 332 | } 333 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /pkg/scheduler/scheduler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scheduler 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 22 | fclient "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions/testing" 23 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 24 | 25 | "github.com/Mirantis/k8s-externalipcontroller/pkg/utils" 26 | "github.com/stretchr/testify/assert" 27 | "github.com/stretchr/testify/mock" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/client-go/kubernetes/fake" 31 | "k8s.io/client-go/pkg/api" 32 | "k8s.io/client-go/pkg/api/v1" 33 | "k8s.io/client-go/tools/cache" 34 | fcache "k8s.io/client-go/tools/cache/testing" 35 | ) 36 | 37 | func TestServiceWatcher(t *testing.T) { 38 | ext := fclient.NewFakeExtClientset() 39 | lw := fcache.NewFakeControllerSource() 40 | stop := make(chan struct{}) 41 | s := ipClaimScheduler{ 42 | DefaultMask: "24", 43 | serviceSource: lw, 44 | ExtensionsClientset: ext, 45 | changeQueue: workqueue.NewQueue(), 46 | } 47 | 48 | ext.Ipclaimpools.On("List", mock.Anything).Return(&extensions.IpClaimPoolList{}, nil) 49 | 50 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 51 | go s.claimChangeWorker() 52 | go s.serviceWatcher(stop) 53 | defer close(stop) 54 | defer s.changeQueue.Close() 55 | 56 | svc := &v1.Service{ 57 | ObjectMeta: metav1.ObjectMeta{Name: "test0"}, 58 | Spec: v1.ServiceSpec{ExternalIPs: []string{"10.10.0.2"}}} 59 | lw.Add(svc) 60 | // let controller process all services 61 | utils.EventualCondition(t, time.Second*1, func() bool { 62 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 63 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 64 | createCall := ext.Ipclaims.Calls[0] 65 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 66 | assert.Equal(t, ipclaim.Spec.Cidr, "10.10.0.2/24", "Unexpected cidr assigned to node") 67 | assert.Equal(t, ipclaim.Metadata.Name, "10-10-0-2-24", "Unexpected name") 68 | 69 | ext.Ipclaims.On("Delete", "10-10-0-2-24", mock.Anything).Return(nil) 70 | lw.Delete(svc) 71 | utils.EventualCondition(t, time.Second*1, func() bool { 72 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 2) 73 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 74 | deleteCall := ext.Ipclaims.Calls[1] 75 | ipclaimName := deleteCall.Arguments[0].(string) 76 | assert.Equal(t, ipclaimName, "10-10-0-2-24", "Unexpected name") 77 | } 78 | 79 | func TestAutoAllocationForServices(t *testing.T) { 80 | ext := fclient.NewFakeExtClientset() 81 | lw := fcache.NewFakeControllerSource() 82 | stop := make(chan struct{}) 83 | svc := v1.Service{ 84 | ObjectMeta: metav1.ObjectMeta{ 85 | Name: "need-alloc-svc", 86 | Annotations: map[string]string{"external-ip": "auto"}, 87 | Namespace: api.NamespaceDefault, 88 | }, 89 | } 90 | 91 | fakeClientset := fake.NewSimpleClientset(&v1.ServiceList{Items: []v1.Service{svc}}) 92 | s := ipClaimScheduler{ 93 | DefaultMask: "24", 94 | serviceSource: lw, 95 | ExtensionsClientset: ext, 96 | Clientset: fakeClientset, 97 | changeQueue: workqueue.NewQueue(), 98 | } 99 | 100 | poolCIDR := "192.168.16.248/29" 101 | poolRanges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 102 | pool := &extensions.IpClaimPool{ 103 | Metadata: metav1.ObjectMeta{Name: "test-pool"}, 104 | Spec: extensions.IpClaimPoolSpec{ 105 | CIDR: poolCIDR, 106 | Ranges: poolRanges, 107 | }, 108 | } 109 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{*pool}} 110 | 111 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 112 | ext.Ipclaimpools.On("Update", mock.Anything).Return(pool, nil) 113 | 114 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 115 | ext.Ipclaims.On("Delete", "192-168-16-250-29", mock.Anything).Return(nil) 116 | ext.Ipclaims.On("Delete", "10-20-0-2-24", mock.Anything).Return(nil) 117 | 118 | go s.claimChangeWorker() 119 | go s.serviceWatcher(stop) 120 | defer close(stop) 121 | defer s.changeQueue.Close() 122 | 123 | lw.Add(&svc) 124 | 125 | utils.EventualCondition(t, time.Second*1, func() bool { 126 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 127 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 128 | 129 | createCall := ext.Ipclaims.Calls[0] 130 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 131 | assert.Equal(t, "192.168.16.250/29", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 132 | assert.Equal(t, "192-168-16-250-29", ipclaim.Metadata.Name, "Unexpected name") 133 | assert.Equal( 134 | t, map[string]string{"ip-pool-name": pool.Metadata.Name}, 135 | ipclaim.Metadata.Labels, 136 | "Labels was not updated by the pool's CIDR") 137 | 138 | poolUpdateCall := ext.Ipclaimpools.Calls[1] 139 | p := poolUpdateCall.Arguments[0].(*extensions.IpClaimPool) 140 | assert.Equal( 141 | t, p.Spec.Allocated, 142 | map[string]string{"192.168.16.250": "192-168-16-250-29"}, 143 | "Allocated was not updated correctly for the pool") 144 | 145 | updatedSvc, _ := fakeClientset.Core().Services(svc.ObjectMeta.Namespace).Get(svc.ObjectMeta.Name, metav1.GetOptions{}) 146 | assert.Equal(t, []string{"192.168.16.250"}, updatedSvc.Spec.ExternalIPs) 147 | 148 | poolList.Items[0].Spec.Allocated = map[string]string{"192.168.16.250": "192-168-16-250-29"} 149 | 150 | changeExternalIPsvc := svc 151 | changeExternalIPsvc.Spec.ExternalIPs = []string{"10.20.0.2"} 152 | delete(changeExternalIPsvc.ObjectMeta.Annotations, "external-ip") 153 | lw.Modify(&changeExternalIPsvc) 154 | 155 | //there should be only 3 calls to Ipclaims at this point: 156 | //1st - creating of 192-168-16-250-29 in AddFunc 157 | //2nd - creating of 10-20-0-2-24 in UpdateFunc 158 | //3rd - deleting of 192-168-16-250-29 in UpdateFunc 159 | //there must not be additional creation of 192-168-16-250-24 - 160 | //ip from the pool but with default mask - at the update of the service 161 | utils.EventualCondition(t, time.Second*1, func() bool { 162 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 3) 163 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 164 | 165 | createCall = ext.Ipclaims.Calls[1] 166 | ipclaim = createCall.Arguments[0].(*extensions.IpClaim) 167 | assert.Equal(t, "10.20.0.2/24", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 168 | assert.Equal(t, "10-20-0-2-24", ipclaim.Metadata.Name, "Unexpected name") 169 | 170 | deleteCall := ext.Ipclaims.Calls[2] 171 | claimName := deleteCall.Arguments[0].(string) 172 | assert.Equal(t, claimName, "192-168-16-250-29", "Unexpected ip claim deleted") 173 | 174 | poolUpdateCall = ext.Ipclaimpools.Calls[4] 175 | p = poolUpdateCall.Arguments[0].(*extensions.IpClaimPool) 176 | assert.NotContains( 177 | t, p.Spec.Allocated, 178 | "192-168-16-250-29", 179 | "Allocated should not contain '192-168-16-250-29'") 180 | } 181 | 182 | func TestClaimNotCreatedIfExternalIPIsAutoAllocated(t *testing.T) { 183 | ext := fclient.NewFakeExtClientset() 184 | lw := fcache.NewFakeControllerSource() 185 | stop := make(chan struct{}) 186 | 187 | poolCIDR := "192.168.16.248/29" 188 | poolRanges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 189 | pool := &extensions.IpClaimPool{ 190 | Metadata: metav1.ObjectMeta{Name: "test-pool"}, 191 | Spec: extensions.IpClaimPoolSpec{ 192 | CIDR: poolCIDR, 193 | Ranges: poolRanges, 194 | Allocated: map[string]string{"192.168.16.250": "192-168-16-250-29"}, 195 | }, 196 | } 197 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{*pool}} 198 | 199 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 200 | 201 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 202 | 203 | s := ipClaimScheduler{ 204 | DefaultMask: "24", 205 | serviceSource: lw, 206 | ExtensionsClientset: ext, 207 | changeQueue: workqueue.NewQueue(), 208 | } 209 | 210 | go s.claimChangeWorker() 211 | go s.serviceWatcher(stop) 212 | defer close(stop) 213 | defer s.changeQueue.Close() 214 | 215 | svc := v1.Service{ 216 | ObjectMeta: metav1.ObjectMeta{ 217 | Name: "need-alloc-svc", 218 | Annotations: map[string]string{"external-ip": "auto"}, 219 | Namespace: api.NamespaceDefault, 220 | }, 221 | Spec: v1.ServiceSpec{ 222 | ExternalIPs: []string{"192.168.16.250", "172.16.0.2"}, 223 | }, 224 | } 225 | lw.Add(&svc) 226 | utils.EventualCondition(t, time.Second*1, func() bool { 227 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 228 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 229 | 230 | createCall := ext.Ipclaims.Calls[0] 231 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 232 | assert.Equal(t, "172.16.0.2/24", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 233 | assert.Equal(t, "172-16-0-2-24", ipclaim.Metadata.Name, "Unexpected name") 234 | } 235 | 236 | func TestAutoAllocatedOnServiceUpdate(t *testing.T) { 237 | ext := fclient.NewFakeExtClientset() 238 | lw := fcache.NewFakeControllerSource() 239 | stop := make(chan struct{}) 240 | 241 | poolCIDR := "192.168.16.248/29" 242 | poolRanges := [][]string{[]string{"192.168.16.250", "192.168.16.252"}} 243 | pool := &extensions.IpClaimPool{ 244 | Metadata: metav1.ObjectMeta{Name: "test-pool"}, 245 | Spec: extensions.IpClaimPoolSpec{ 246 | CIDR: poolCIDR, 247 | Ranges: poolRanges, 248 | }, 249 | } 250 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{*pool}} 251 | 252 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 253 | ext.Ipclaimpools.On("Update", mock.Anything).Return(pool, nil) 254 | 255 | ext.Ipclaims.On("Create", mock.Anything).Return(nil) 256 | 257 | svc := v1.Service{ 258 | ObjectMeta: metav1.ObjectMeta{ 259 | Name: "need-alloc-svc", 260 | Namespace: api.NamespaceDefault, 261 | }, 262 | Spec: v1.ServiceSpec{ 263 | ExternalIPs: []string{"172.16.0.2"}, 264 | }, 265 | } 266 | 267 | fakeClientset := fake.NewSimpleClientset(&v1.ServiceList{Items: []v1.Service{svc}}) 268 | 269 | s := ipClaimScheduler{ 270 | DefaultMask: "24", 271 | serviceSource: lw, 272 | ExtensionsClientset: ext, 273 | Clientset: fakeClientset, 274 | changeQueue: workqueue.NewQueue(), 275 | } 276 | 277 | go s.claimChangeWorker() 278 | go s.serviceWatcher(stop) 279 | defer close(stop) 280 | defer s.changeQueue.Close() 281 | 282 | lw.Add(&svc) 283 | 284 | utils.EventualCondition(t, time.Second*1, func() bool { 285 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 286 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 287 | 288 | annotatedSvc := svc 289 | annotatedSvc.ObjectMeta.Annotations = map[string]string{"external-ip": "auto"} 290 | lw.Modify(&annotatedSvc) 291 | 292 | //one extra create for 172-16-0-2-24 as the service is double processed 293 | utils.EventualCondition(t, time.Second*1, func() bool { 294 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 3) 295 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 296 | 297 | createCall := ext.Ipclaims.Calls[2] 298 | ipclaim := createCall.Arguments[0].(*extensions.IpClaim) 299 | assert.Equal(t, "192.168.16.250/29", ipclaim.Spec.Cidr, "Unexpected cidr assigned to node") 300 | assert.Equal(t, "192-168-16-250-29", ipclaim.Metadata.Name, "Unexpected name") 301 | 302 | updatedSvc, _ := fakeClientset.Core().Services(svc.ObjectMeta.Namespace).Get(svc.ObjectMeta.Name, metav1.GetOptions{}) 303 | assert.Contains(t, updatedSvc.Spec.ExternalIPs, "192.168.16.250") 304 | } 305 | 306 | func TestClaimWatcher(t *testing.T) { 307 | ext := fclient.NewFakeExtClientset() 308 | lw := fcache.NewFakeControllerSource() 309 | fss := cache.NewStore(cache.MetaNamespaceKeyFunc) 310 | 311 | svc := v1.Service{ 312 | ObjectMeta: metav1.ObjectMeta{ 313 | Name: "some-svc", 314 | Namespace: api.NamespaceDefault, 315 | }, 316 | Spec: v1.ServiceSpec{ 317 | ExternalIPs: []string{"10.10.0.2"}, 318 | }, 319 | } 320 | fss.Add(&svc) 321 | 322 | stop := make(chan struct{}) 323 | s := ipClaimScheduler{ 324 | claimSource: lw, 325 | serviceStore: fss, 326 | ExtensionsClientset: ext, 327 | liveIpNodes: make(map[string]struct{}), 328 | queue: workqueue.NewQueue(), 329 | changeQueue: workqueue.NewQueue(), 330 | } 331 | s.getNode = s.getFairNode 332 | 333 | poolList := &extensions.IpClaimPoolList{Items: []extensions.IpClaimPool{}} 334 | ext.Ipclaimpools.On("List", mock.Anything).Return(poolList, nil) 335 | 336 | go s.worker() 337 | go s.claimChangeWorker() 338 | go s.claimWatcher(stop) 339 | defer close(stop) 340 | defer s.queue.Close() 341 | defer s.changeQueue.Close() 342 | 343 | ctrl := false 344 | ownerRef := metav1.OwnerReference{APIVersion: "v1", Kind: "Service", Name: "some-svc", UID: types.UID("default/some-svc"), Controller: &ctrl} 345 | claim := &extensions.IpClaim{ 346 | Metadata: metav1.ObjectMeta{Name: "10.10.0.2-24", OwnerReferences: []metav1.OwnerReference{ownerRef}}, 347 | Spec: extensions.IpClaimSpec{Cidr: "10.10.0.2/24"}, 348 | } 349 | lw.Add(claim) 350 | 351 | ipnodesList := &extensions.IpNodeList{ 352 | Items: []extensions.IpNode{ 353 | { 354 | Metadata: metav1.ObjectMeta{Name: "first"}, 355 | }, 356 | }, 357 | } 358 | for _, node := range ipnodesList.Items { 359 | s.liveIpNodes[node.Metadata.Name] = struct{}{} 360 | } 361 | 362 | ext.Ipnodes.On("List", mock.Anything).Return(ipnodesList, nil) 363 | ext.Ipclaims.On("Update", mock.Anything).Return(nil) 364 | utils.EventualCondition(t, time.Second*1, func() bool { 365 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 366 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 367 | updatedClaim := ext.Ipclaims.Calls[0].Arguments[0].(*extensions.IpClaim) 368 | assert.Equal(t, updatedClaim.Metadata.Labels, map[string]string{"ipnode": "first"}, 369 | "Labels should be set to scheduled node") 370 | assert.Equal(t, updatedClaim.Spec.NodeName, "first", "NodeName should be set to scheduled node") 371 | } 372 | 373 | func TestMonitorIpNodes(t *testing.T) { 374 | ext := fclient.NewFakeExtClientset() 375 | stop := make(chan struct{}) 376 | ticker := make(chan time.Time, 2) 377 | for i := 0; i < 2; i++ { 378 | ticker <- time.Time{} 379 | } 380 | s := ipClaimScheduler{ 381 | ExtensionsClientset: ext, 382 | liveIpNodes: make(map[string]struct{}), 383 | observedGeneration: make(map[string]int64), 384 | queue: workqueue.NewQueue(), 385 | } 386 | ipnodesList := &extensions.IpNodeList{ 387 | Items: []extensions.IpNode{ 388 | { 389 | Metadata: metav1.ObjectMeta{ 390 | Name: "first", 391 | }, 392 | Revision: 555, 393 | }, 394 | }, 395 | } 396 | ipclaimsList := &extensions.IpClaimList{ 397 | Items: []extensions.IpClaim{ 398 | { 399 | Metadata: metav1.ObjectMeta{ 400 | Name: "10.10.0.1-24", 401 | Labels: map[string]string{"ipnode": "first"}, 402 | }, 403 | Spec: extensions.IpClaimSpec{ 404 | Cidr: "10.10.0.1/24", 405 | NodeName: "first", 406 | }, 407 | }, 408 | { 409 | Metadata: metav1.ObjectMeta{ 410 | Name: "10.10.0.2-24", 411 | Labels: map[string]string{"ipnode": "first"}, 412 | }, 413 | Spec: extensions.IpClaimSpec{ 414 | Cidr: "10.10.0.2/24", 415 | NodeName: "first", 416 | }, 417 | }, 418 | }, 419 | } 420 | ext.Ipnodes.On("List", mock.Anything).Return(ipnodesList, nil).Twice() 421 | ext.Ipclaims.On("List", mock.Anything).Return(ipclaimsList, nil) 422 | go s.monitorIPNodes(stop, ticker) 423 | defer close(stop) 424 | utils.EventualCondition(t, time.Second*1, func() bool { 425 | return assert.ObjectsAreEqual(len(ext.Ipnodes.Calls), 2) 426 | }, "Unexpected call count to ipnodes", ext.Ipnodes.Calls) 427 | utils.EventualCondition(t, time.Second*1, func() bool { 428 | return assert.ObjectsAreEqual(len(ext.Ipclaims.Calls), 1) 429 | }, "Unexpected call count to ipclaims", ext.Ipclaims.Calls) 430 | assert.Equal(t, s.isLive("first"), false, "first node shouldn't be considered live") 431 | } 432 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 8bb4416e86a28d0a658fc033718394f0041835f5969ae10c02dc0ff08ceda1bb 2 | updated: 2017-10-12T14:43:36.430701013+03:00 3 | imports: 4 | - name: cloud.google.com/go 5 | version: 3b1ae45394a234c385be014e9a488f2bb6eef821 6 | subpackages: 7 | - compute/metadata 8 | - internal 9 | - name: github.com/Azure/go-ansiterm 10 | version: 70b2c90b260171e829f1ebd7c17f600c11858dbe 11 | subpackages: 12 | - winterm 13 | - name: github.com/blang/semver 14 | version: 31b736133b98f26d5e078ec9eb591666edfd091f 15 | - name: github.com/coreos/etcd 16 | version: 83347907774bf36cbb261c594a32fd7b0f5dd9f6 17 | subpackages: 18 | - client 19 | - name: github.com/coreos/go-oidc 20 | version: be73733bb8cc830d0205609b95d125215f8e9c70 21 | subpackages: 22 | - http 23 | - jose 24 | - key 25 | - oauth2 26 | - oidc 27 | - name: github.com/coreos/pkg 28 | version: fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 29 | subpackages: 30 | - health 31 | - httputil 32 | - timeutil 33 | - name: github.com/cpuguy83/go-md2man 34 | version: 23709d0847197db6021a51fdb193e66e9222d4e7 35 | subpackages: 36 | - md2man 37 | - name: github.com/davecgh/go-spew 38 | version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d 39 | subpackages: 40 | - spew 41 | - name: github.com/docker/distribution 42 | version: cd27f179f2c10c5d300e6d09025b538c475b0d51 43 | subpackages: 44 | - digest 45 | - reference 46 | - name: github.com/docker/docker 47 | version: b9f10c951893f9a00865890a5232e85d770c1087 48 | subpackages: 49 | - pkg/jsonlog 50 | - pkg/jsonmessage 51 | - pkg/longpath 52 | - pkg/mount 53 | - pkg/stdcopy 54 | - pkg/symlink 55 | - pkg/system 56 | - pkg/term 57 | - pkg/term/windows 58 | - name: github.com/docker/go-units 59 | version: 0bbddae09c5a5419a8c6dcdd7ff90da3d450393b 60 | - name: github.com/docker/spdystream 61 | version: 449fdfce4d962303d702fec724ef0ad181c92528 62 | subpackages: 63 | - spdy 64 | - name: github.com/emicklei/go-restful 65 | version: 777bb3f19bcafe2575ffb2a3e46af92509ae9594 66 | subpackages: 67 | - log 68 | - swagger 69 | - name: github.com/emicklei/go-restful-swagger12 70 | version: dcef7f55730566d41eae5db10e7d6981829720f6 71 | - name: github.com/ghodss/yaml 72 | version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee 73 | - name: github.com/go-openapi/analysis 74 | version: b44dc874b601d9e4e2f6e19140e794ba24bead3b 75 | - name: github.com/go-openapi/jsonpointer 76 | version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 77 | - name: github.com/go-openapi/jsonreference 78 | version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272 79 | - name: github.com/go-openapi/loads 80 | version: 18441dfa706d924a39a030ee2c3b1d8d81917b38 81 | - name: github.com/go-openapi/spec 82 | version: 6aced65f8501fe1217321abf0749d354824ba2ff 83 | - name: github.com/go-openapi/swag 84 | version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 85 | - name: github.com/gogo/protobuf 86 | version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 87 | subpackages: 88 | - proto 89 | - sortkeys 90 | - name: github.com/golang/glog 91 | version: 44145f04b68cf362d9c4df2182967c2275eaefed 92 | - name: github.com/golang/groupcache 93 | version: 02826c3e79038b59d737d3b1c0a1d937f71a4433 94 | subpackages: 95 | - lru 96 | - name: github.com/golang/protobuf 97 | version: 4bd1920723d7b7c925de087aa32e2187708897f7 98 | subpackages: 99 | - proto 100 | - name: github.com/google/gofuzz 101 | version: 44d81051d367757e1c7c6a5a86423ece9afcf63c 102 | - name: github.com/google/gopacket 103 | version: b83f94714c36e30ce851be1d5a0a5226f9f1bca4 104 | subpackages: 105 | - layers 106 | - pcap 107 | - name: github.com/hashicorp/golang-lru 108 | version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 109 | subpackages: 110 | - simplelru 111 | - name: github.com/howeyc/gopass 112 | version: bf9dde6d0d2c004a008c27aaee91170c786f6db8 113 | - name: github.com/imdario/mergo 114 | version: 6633656539c1639d9d78127b7d47c622b5d7b6dc 115 | - name: github.com/inconshreveable/mousetrap 116 | version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 117 | - name: github.com/jonboulle/clockwork 118 | version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982 119 | - name: github.com/juju/ratelimit 120 | version: 5b9ff866471762aa2ab2dced63c9fb6f53921342 121 | - name: github.com/mailru/easyjson 122 | version: d5b7844b561a7bc640052f1b935f7b800330d7e0 123 | subpackages: 124 | - buffer 125 | - jlexer 126 | - jwriter 127 | - name: github.com/mitchellh/go-wordwrap 128 | version: ad45545899c7b13c020ea92b2072220eefad42b8 129 | - name: github.com/onsi/ginkgo 130 | version: 74c678d97c305753605c338c6c78c49ec104b5e7 131 | subpackages: 132 | - config 133 | - internal/codelocation 134 | - internal/containernode 135 | - internal/failer 136 | - internal/leafnodes 137 | - internal/remote 138 | - internal/spec 139 | - internal/specrunner 140 | - internal/suite 141 | - internal/testingtproxy 142 | - internal/writer 143 | - reporters 144 | - reporters/stenographer 145 | - types 146 | - name: github.com/onsi/gomega 147 | version: a78ae492d53aad5a7a232d0d0462c14c400e3ee7 148 | subpackages: 149 | - format 150 | - internal/assertion 151 | - internal/asyncassertion 152 | - internal/testingtsupport 153 | - matchers 154 | - matchers/support/goraph/bipartitegraph 155 | - matchers/support/goraph/edge 156 | - matchers/support/goraph/node 157 | - matchers/support/goraph/util 158 | - types 159 | - name: github.com/pborman/uuid 160 | version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 161 | - name: github.com/pmezard/go-difflib 162 | version: 792786c7400a136282c1664665ae0a8db921c6c2 163 | subpackages: 164 | - difflib 165 | - name: github.com/PuerkitoBio/purell 166 | version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 167 | - name: github.com/PuerkitoBio/urlesc 168 | version: 5bd2802263f21d8788851d5305584c82a5c75d7e 169 | - name: github.com/russross/blackfriday 170 | version: 300106c228d52c8941d4b3de6054a6062a86dda3 171 | - name: github.com/shurcooL/sanitized_anchor_name 172 | version: 10ef21a441db47d8b13ebcc5fd2310f636973c77 173 | - name: github.com/Sirupsen/logrus 174 | version: 51fe59aca108dc5680109e7b2051cbdcfa5a253c 175 | - name: github.com/spf13/cobra 176 | version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b 177 | - name: github.com/spf13/pflag 178 | version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 179 | - name: github.com/stretchr/objx 180 | version: 1a9d0bb9f541897e62256577b352fdbc1fb4fd94 181 | - name: github.com/stretchr/testify 182 | version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 183 | subpackages: 184 | - assert 185 | - mock 186 | - name: github.com/ugorji/go 187 | version: ded73eae5db7e7a0ef6f55aace87a2873c5d2b74 188 | subpackages: 189 | - codec 190 | - name: github.com/vishvananda/netlink 191 | version: 1e2e08e8a2dcdacaae3f14ac44c5cfa31361f270 192 | subpackages: 193 | - nl 194 | - name: golang.org/x/crypto 195 | version: d172538b2cfce0c13cee31e647d0367aa8cd2486 196 | subpackages: 197 | - ssh/terminal 198 | - name: golang.org/x/net 199 | version: f2499483f923065a842d38eb4c7f1927e6fc6e6d 200 | subpackages: 201 | - context 202 | - context/ctxhttp 203 | - http2 204 | - http2/hpack 205 | - idna 206 | - lex/httplex 207 | - websocket 208 | - name: golang.org/x/oauth2 209 | version: a6bd8cefa1811bd24b86f8902872e4e8225f74c4 210 | subpackages: 211 | - google 212 | - internal 213 | - jws 214 | - jwt 215 | - name: golang.org/x/sys 216 | version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 217 | subpackages: 218 | - unix 219 | - name: golang.org/x/text 220 | version: 2910a502d2bf9e43193af9d68ca516529614eed3 221 | subpackages: 222 | - cases 223 | - internal/tag 224 | - language 225 | - runes 226 | - secure/bidirule 227 | - secure/precis 228 | - transform 229 | - unicode/bidi 230 | - unicode/norm 231 | - width 232 | - name: google.golang.org/appengine 233 | version: 4f7eeb5305a4ba1966344836ba4af9996b7b4e05 234 | subpackages: 235 | - internal 236 | - internal/app_identity 237 | - internal/base 238 | - internal/datastore 239 | - internal/log 240 | - internal/modules 241 | - internal/remote_api 242 | - internal/urlfetch 243 | - urlfetch 244 | - name: gopkg.in/inf.v0 245 | version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 246 | - name: gopkg.in/yaml.v2 247 | version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 248 | - name: k8s.io/api 249 | version: e24ed681f56165befbea50a24f7e9f1c079ac366 250 | - name: k8s.io/apiextensions-apiserver 251 | version: 3eb4c5fefbe7ad866747f78c7e4994cb555170c6 252 | - name: k8s.io/apimachinery 253 | version: 1fd2e63a9a370677308a42f24fd40c86438afddf 254 | subpackages: 255 | - pkg/api/equality 256 | - pkg/api/errors 257 | - pkg/api/meta 258 | - pkg/api/resource 259 | - pkg/apimachinery 260 | - pkg/apimachinery/announced 261 | - pkg/apimachinery/registered 262 | - pkg/apis/meta/v1 263 | - pkg/apis/meta/v1/unstructured 264 | - pkg/apis/meta/v1alpha1 265 | - pkg/conversion 266 | - pkg/conversion/queryparams 267 | - pkg/conversion/unstructured 268 | - pkg/fields 269 | - pkg/labels 270 | - pkg/openapi 271 | - pkg/runtime 272 | - pkg/runtime/schema 273 | - pkg/runtime/serializer 274 | - pkg/runtime/serializer/json 275 | - pkg/runtime/serializer/protobuf 276 | - pkg/runtime/serializer/recognizer 277 | - pkg/runtime/serializer/streaming 278 | - pkg/runtime/serializer/versioning 279 | - pkg/selection 280 | - pkg/types 281 | - pkg/util/cache 282 | - pkg/util/clock 283 | - pkg/util/diff 284 | - pkg/util/errors 285 | - pkg/util/framer 286 | - pkg/util/intstr 287 | - pkg/util/json 288 | - pkg/util/net 289 | - pkg/util/rand 290 | - pkg/util/runtime 291 | - pkg/util/sets 292 | - pkg/util/validation 293 | - pkg/util/validation/field 294 | - pkg/util/wait 295 | - pkg/util/yaml 296 | - pkg/version 297 | - pkg/watch 298 | - third_party/forked/golang/reflect 299 | - name: k8s.io/client-go 300 | version: d92e8497f71b7b4e0494e5bd204b48d34bd6f254 301 | subpackages: 302 | - discovery 303 | - discovery/fake 304 | - kubernetes 305 | - kubernetes/fake 306 | - kubernetes/scheme 307 | - kubernetes/typed/admissionregistration/v1alpha1 308 | - kubernetes/typed/admissionregistration/v1alpha1/fake 309 | - kubernetes/typed/apps/v1beta1 310 | - kubernetes/typed/apps/v1beta1/fake 311 | - kubernetes/typed/authentication/v1 312 | - kubernetes/typed/authentication/v1/fake 313 | - kubernetes/typed/authentication/v1beta1 314 | - kubernetes/typed/authentication/v1beta1/fake 315 | - kubernetes/typed/authorization/v1 316 | - kubernetes/typed/authorization/v1/fake 317 | - kubernetes/typed/authorization/v1beta1 318 | - kubernetes/typed/authorization/v1beta1/fake 319 | - kubernetes/typed/autoscaling/v1 320 | - kubernetes/typed/autoscaling/v1/fake 321 | - kubernetes/typed/autoscaling/v2alpha1 322 | - kubernetes/typed/autoscaling/v2alpha1/fake 323 | - kubernetes/typed/batch/v1 324 | - kubernetes/typed/batch/v1/fake 325 | - kubernetes/typed/batch/v2alpha1 326 | - kubernetes/typed/batch/v2alpha1/fake 327 | - kubernetes/typed/certificates/v1beta1 328 | - kubernetes/typed/certificates/v1beta1/fake 329 | - kubernetes/typed/core/v1 330 | - kubernetes/typed/core/v1/fake 331 | - kubernetes/typed/extensions/v1beta1 332 | - kubernetes/typed/extensions/v1beta1/fake 333 | - kubernetes/typed/networking/v1 334 | - kubernetes/typed/networking/v1/fake 335 | - kubernetes/typed/policy/v1beta1 336 | - kubernetes/typed/policy/v1beta1/fake 337 | - kubernetes/typed/rbac/v1alpha1 338 | - kubernetes/typed/rbac/v1alpha1/fake 339 | - kubernetes/typed/rbac/v1beta1 340 | - kubernetes/typed/rbac/v1beta1/fake 341 | - kubernetes/typed/settings/v1alpha1 342 | - kubernetes/typed/settings/v1alpha1/fake 343 | - kubernetes/typed/storage/v1 344 | - kubernetes/typed/storage/v1/fake 345 | - kubernetes/typed/storage/v1beta1 346 | - kubernetes/typed/storage/v1beta1/fake 347 | - pkg/api 348 | - pkg/api/v1 349 | - pkg/api/v1/ref 350 | - pkg/apis/admissionregistration 351 | - pkg/apis/admissionregistration/v1alpha1 352 | - pkg/apis/apps 353 | - pkg/apis/apps/v1beta1 354 | - pkg/apis/authentication 355 | - pkg/apis/authentication/v1 356 | - pkg/apis/authentication/v1beta1 357 | - pkg/apis/authorization 358 | - pkg/apis/authorization/v1 359 | - pkg/apis/authorization/v1beta1 360 | - pkg/apis/autoscaling 361 | - pkg/apis/autoscaling/v1 362 | - pkg/apis/autoscaling/v2alpha1 363 | - pkg/apis/batch 364 | - pkg/apis/batch/v1 365 | - pkg/apis/batch/v2alpha1 366 | - pkg/apis/certificates 367 | - pkg/apis/certificates/v1beta1 368 | - pkg/apis/extensions 369 | - pkg/apis/extensions/v1beta1 370 | - pkg/apis/networking 371 | - pkg/apis/networking/v1 372 | - pkg/apis/policy 373 | - pkg/apis/policy/v1beta1 374 | - pkg/apis/rbac 375 | - pkg/apis/rbac/v1alpha1 376 | - pkg/apis/rbac/v1beta1 377 | - pkg/apis/settings 378 | - pkg/apis/settings/v1alpha1 379 | - pkg/apis/storage 380 | - pkg/apis/storage/v1 381 | - pkg/apis/storage/v1beta1 382 | - pkg/runtime 383 | - pkg/util 384 | - pkg/util/parsers 385 | - pkg/version 386 | - pkg/watch 387 | - rest 388 | - rest/watch 389 | - testing 390 | - tools/auth 391 | - tools/cache 392 | - tools/cache/testing 393 | - tools/clientcmd 394 | - tools/clientcmd/api 395 | - tools/clientcmd/api/latest 396 | - tools/clientcmd/api/v1 397 | - tools/metrics 398 | - transport 399 | - util/cert 400 | - util/flowcontrol 401 | - util/homedir 402 | - util/integer 403 | - name: k8s.io/kubernetes 404 | version: 42fe4ab0270e44c750d77c682e2fcab394aeb392 405 | subpackages: 406 | - pkg/api 407 | - pkg/api/errors 408 | - pkg/api/install 409 | - pkg/api/meta 410 | - pkg/api/meta/metatypes 411 | - pkg/api/resource 412 | - pkg/api/unversioned 413 | - pkg/api/v1 414 | - pkg/api/validation/path 415 | - pkg/apimachinery 416 | - pkg/apimachinery/announced 417 | - pkg/apimachinery/registered 418 | - pkg/apis/apps 419 | - pkg/apis/apps/install 420 | - pkg/apis/apps/v1beta1 421 | - pkg/apis/authentication 422 | - pkg/apis/authentication/install 423 | - pkg/apis/authentication/v1beta1 424 | - pkg/apis/authorization 425 | - pkg/apis/authorization/install 426 | - pkg/apis/authorization/v1beta1 427 | - pkg/apis/autoscaling 428 | - pkg/apis/autoscaling/install 429 | - pkg/apis/autoscaling/v1 430 | - pkg/apis/batch 431 | - pkg/apis/batch/install 432 | - pkg/apis/batch/v1 433 | - pkg/apis/batch/v2alpha1 434 | - pkg/apis/certificates 435 | - pkg/apis/certificates/install 436 | - pkg/apis/certificates/v1alpha1 437 | - pkg/apis/componentconfig 438 | - pkg/apis/componentconfig/install 439 | - pkg/apis/componentconfig/v1alpha1 440 | - pkg/apis/extensions 441 | - pkg/apis/extensions/install 442 | - pkg/apis/extensions/v1beta1 443 | - pkg/apis/policy 444 | - pkg/apis/policy/install 445 | - pkg/apis/policy/v1beta1 446 | - pkg/apis/rbac 447 | - pkg/apis/rbac/install 448 | - pkg/apis/rbac/v1alpha1 449 | - pkg/apis/storage 450 | - pkg/apis/storage/install 451 | - pkg/apis/storage/v1beta1 452 | - pkg/auth/user 453 | - pkg/client/clientset_generated/internalclientset 454 | - pkg/client/clientset_generated/internalclientset/typed/apps/internalversion 455 | - pkg/client/clientset_generated/internalclientset/typed/authentication/internalversion 456 | - pkg/client/clientset_generated/internalclientset/typed/authorization/internalversion 457 | - pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion 458 | - pkg/client/clientset_generated/internalclientset/typed/batch/internalversion 459 | - pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion 460 | - pkg/client/clientset_generated/internalclientset/typed/core/internalversion 461 | - pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion 462 | - pkg/client/clientset_generated/internalclientset/typed/policy/internalversion 463 | - pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion 464 | - pkg/client/clientset_generated/internalclientset/typed/storage/internalversion 465 | - pkg/client/leaderelection 466 | - pkg/client/leaderelection/resourcelock 467 | - pkg/client/metrics 468 | - pkg/client/record 469 | - pkg/client/restclient 470 | - pkg/client/transport 471 | - pkg/client/typed/discovery 472 | - pkg/client/unversioned/clientcmd/api 473 | - pkg/client/unversioned/remotecommand 474 | - pkg/conversion 475 | - pkg/conversion/queryparams 476 | - pkg/fields 477 | - pkg/genericapiserver/openapi/common 478 | - pkg/httplog 479 | - pkg/kubelet/qos 480 | - pkg/kubelet/server/remotecommand 481 | - pkg/kubelet/types 482 | - pkg/labels 483 | - pkg/master/ports 484 | - pkg/runtime 485 | - pkg/runtime/serializer 486 | - pkg/runtime/serializer/json 487 | - pkg/runtime/serializer/protobuf 488 | - pkg/runtime/serializer/recognizer 489 | - pkg/runtime/serializer/streaming 490 | - pkg/runtime/serializer/versioning 491 | - pkg/selection 492 | - pkg/types 493 | - pkg/util 494 | - pkg/util/cert 495 | - pkg/util/clock 496 | - pkg/util/config 497 | - pkg/util/errors 498 | - pkg/util/exec 499 | - pkg/util/flag 500 | - pkg/util/flowcontrol 501 | - pkg/util/framer 502 | - pkg/util/httpstream 503 | - pkg/util/httpstream/spdy 504 | - pkg/util/integer 505 | - pkg/util/interrupt 506 | - pkg/util/intstr 507 | - pkg/util/json 508 | - pkg/util/labels 509 | - pkg/util/net 510 | - pkg/util/parsers 511 | - pkg/util/rand 512 | - pkg/util/runtime 513 | - pkg/util/sets 514 | - pkg/util/strategicpatch 515 | - pkg/util/term 516 | - pkg/util/uuid 517 | - pkg/util/validation 518 | - pkg/util/validation/field 519 | - pkg/util/wait 520 | - pkg/util/wsstream 521 | - pkg/util/yaml 522 | - pkg/version 523 | - pkg/watch 524 | - pkg/watch/versioned 525 | - plugin/pkg/client/auth 526 | - plugin/pkg/client/auth/gcp 527 | - plugin/pkg/client/auth/oidc 528 | - third_party/forked/golang/json 529 | - third_party/forked/golang/netutil 530 | - third_party/forked/golang/reflect 531 | testImports: [] 532 | -------------------------------------------------------------------------------- /pkg/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Mirantis 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scheduler 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "strings" 21 | "sync" 22 | "time" 23 | 24 | "github.com/Mirantis/k8s-externalipcontroller/pkg/extensions" 25 | "github.com/Mirantis/k8s-externalipcontroller/pkg/workqueue" 26 | 27 | "github.com/golang/glog" 28 | apierrors "k8s.io/apimachinery/pkg/api/errors" 29 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | "k8s.io/apimachinery/pkg/fields" 31 | "k8s.io/apimachinery/pkg/labels" 32 | "k8s.io/apimachinery/pkg/runtime" 33 | "k8s.io/apimachinery/pkg/types" 34 | "k8s.io/apimachinery/pkg/watch" 35 | "k8s.io/client-go/kubernetes" 36 | "k8s.io/client-go/pkg/api" 37 | "k8s.io/client-go/pkg/api/v1" 38 | "k8s.io/client-go/rest" 39 | "k8s.io/client-go/tools/cache" 40 | ) 41 | 42 | const ( 43 | AutoExternalAnnotationKey = "external-ip" 44 | AutoExternalAnnotationValue = "auto" 45 | ) 46 | 47 | func NewIPClaimScheduler(config *rest.Config, mask string, monitorInterval time.Duration, nodeFilter string) (*ipClaimScheduler, error) { 48 | clientset, err := kubernetes.NewForConfig(config) 49 | if err != nil { 50 | return nil, err 51 | } 52 | ext, err := extensions.WrapClientsetWithExtensions(clientset, config) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | serviceSource := &cache.ListWatch{ 58 | ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 59 | return clientset.Core().Services(api.NamespaceAll).List(options) 60 | }, 61 | WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 62 | return clientset.Core().Services(api.NamespaceAll).Watch(options) 63 | }, 64 | } 65 | 66 | claimSource := cache.NewListWatchFromClient(ext.Client, "ipclaims", api.NamespaceAll, fields.Everything()) 67 | scheduler := ipClaimScheduler{ 68 | Config: config, 69 | Clientset: clientset, 70 | ExtensionsClientset: ext, 71 | DefaultMask: mask, 72 | 73 | monitorPeriod: monitorInterval, 74 | serviceSource: serviceSource, 75 | claimSource: claimSource, 76 | 77 | observedGeneration: make(map[string]int64), 78 | liveIpNodes: make(map[string]struct{}), 79 | 80 | queue: workqueue.NewQueue(), 81 | changeQueue: workqueue.NewQueue(), 82 | } 83 | 84 | switch nodeFilter { 85 | case "fair": 86 | scheduler.getNode = scheduler.getFairNode 87 | case "first-alive": 88 | scheduler.getNode = scheduler.getFirstAliveNode 89 | default: 90 | return nil, errors.New("Incorrect node filter is provided") 91 | } 92 | 93 | return &scheduler, nil 94 | } 95 | 96 | type nodeFilter func([]*extensions.IpNode) *extensions.IpNode 97 | 98 | type ipClaimScheduler struct { 99 | Config *rest.Config 100 | Clientset kubernetes.Interface 101 | ExtensionsClientset extensions.ExtensionsClientset 102 | DefaultMask string 103 | 104 | serviceSource cache.ListerWatcher 105 | claimSource cache.ListerWatcher 106 | 107 | monitorPeriod time.Duration 108 | observedGeneration map[string]int64 109 | liveSync sync.Mutex 110 | liveIpNodes map[string]struct{} 111 | 112 | claimStore cache.Store 113 | serviceStore cache.Store 114 | 115 | getNode nodeFilter 116 | 117 | queue workqueue.QueueType 118 | changeQueue workqueue.QueueType 119 | } 120 | 121 | func (s *ipClaimScheduler) Run(stop chan struct{}) { 122 | glog.V(3).Infof("Starting monitor goroutine.") 123 | go s.monitorIPNodes(stop, time.Tick(s.monitorPeriod)) 124 | // let's give controllers some time to register themselves after scheduler restart 125 | // TODO(dshulyak) consider to run monitor both for leaders/non-leaders 126 | time.Sleep(s.monitorPeriod) 127 | glog.V(3).Infof("Starting all other worker goroutines.") 128 | go s.worker() 129 | go s.claimChangeWorker() 130 | go s.serviceWatcher(stop) 131 | go s.claimWatcher(stop) 132 | <-stop 133 | s.queue.Close() 134 | s.changeQueue.Close() 135 | } 136 | 137 | // serviceWatcher creates/deletes IPClaim based on requirements from 138 | // service 139 | func (s *ipClaimScheduler) serviceWatcher(stop chan struct{}) { 140 | store, controller := cache.NewInformer( 141 | s.serviceSource, 142 | &v1.Service{}, 143 | 0, 144 | cache.ResourceEventHandlerFuncs{ 145 | AddFunc: func(obj interface{}) { 146 | svc := obj.(*v1.Service) 147 | s.processExternalIPs(svc) 148 | }, 149 | UpdateFunc: func(old, cur interface{}) { 150 | curSvc := cur.(*v1.Service) 151 | s.processExternalIPs(curSvc) 152 | 153 | oldSvc := old.(*v1.Service) 154 | s.processOldService(oldSvc) 155 | }, 156 | DeleteFunc: func(obj interface{}) { 157 | svc := obj.(*v1.Service) 158 | s.processOldService(svc) 159 | }, 160 | }, 161 | ) 162 | s.serviceStore = store 163 | controller.Run(stop) 164 | } 165 | 166 | // we must take into account that loss of events and double processing of 167 | // service objects might occur (due the way the object cache works); thus 168 | // auto allocation must be done only in case a service is properly annotated 169 | // and there is no already auto allocated IP for it 170 | func (s *ipClaimScheduler) processExternalIPs(svc *v1.Service) { 171 | foundAuto := false 172 | 173 | pools, err := s.ExtensionsClientset.IPClaimPools().List(metav1.ListOptions{}) 174 | if err != nil { 175 | glog.Errorf("Error retrieving list of IP pools. Details: %v", err) 176 | } 177 | 178 | glog.V(2).Infof("Processing svc %s with external ips %v", svc.Name, svc.Spec.ExternalIPs) 179 | for _, ip := range svc.Spec.ExternalIPs { 180 | glog.V(2).Infof( 181 | "Check IP %s of a service %s for an intersection with pools: %v", 182 | ip, svc.Name, pools) 183 | if p := poolByAllocatedIP(ip, pools); p != nil { 184 | foundAuto = true 185 | continue 186 | } 187 | s.addClaimChangeRequest(makeIPClaim(ip, s.DefaultMask, svc), cache.Added) 188 | } 189 | if foundAuto { 190 | return 191 | } 192 | 193 | if annotated := checkAnnotation(svc); annotated { 194 | s.autoAllocateExternalIP(svc, pools, false) 195 | } else if svc.Spec.Type == v1.ServiceTypeLoadBalancer { 196 | glog.Infof("Allocate external ip for service %s/%s based on LoadBalancer type request", 197 | svc.Namespace, svc.Name) 198 | s.autoAllocateExternalIP(svc, pools, true) 199 | } 200 | } 201 | 202 | func poolByAllocatedIP(ip string, poolList *extensions.IpClaimPoolList) *extensions.IpClaimPool { 203 | for _, pool := range poolList.Items { 204 | if _, exists := pool.Spec.Allocated[ip]; exists { 205 | return &pool 206 | } 207 | } 208 | return nil 209 | } 210 | 211 | func (s *ipClaimScheduler) autoAllocateExternalIP(svc *v1.Service, poolList *extensions.IpClaimPoolList, setLBIp bool) { 212 | glog.V(5).Infof("Try to auto allocate external IP for service '%v'", svc.ObjectMeta.Name) 213 | 214 | var freeIP string 215 | var pool extensions.IpClaimPool 216 | 217 | for _, p := range poolList.Items { 218 | ip, err := p.AvailableIP() 219 | if err != nil { 220 | glog.Errorf( 221 | "Fail to retrieve free IP from the pool '%v'; skipping to a next one. Details: %v", 222 | p.Metadata.Name, err) 223 | continue 224 | } 225 | freeIP = ip 226 | pool = p 227 | break 228 | } 229 | 230 | if len(freeIP) == 0 { 231 | glog.Errorf( 232 | "Fail to provide external IP for service '%v'. All pools are exhausted.", 233 | svc.ObjectMeta.Name) 234 | return 235 | } 236 | 237 | mask := strings.Split(pool.Spec.CIDR, "/")[1] 238 | 239 | ipclaim := makeIPClaim(freeIP, mask, svc) 240 | ipclaim.Metadata.SetLabels( 241 | map[string]string{"ip-pool-name": pool.Metadata.Name}) 242 | 243 | s.addClaimChangeRequest(ipclaim, cache.Added) 244 | 245 | err := updatePoolAllocation(s.ExtensionsClientset, &pool, freeIP, ipclaim.Metadata.Name) 246 | if err != nil { 247 | glog.Errorf("Unable to update IP pool's '%v' allocation. Details: %v", 248 | pool.Metadata.Name, err) 249 | } 250 | glog.V(5).Infof( 251 | "Try to update externalIPs list of service '%v' with IP address '%v'", 252 | svc.ObjectMeta.Name, freeIP, 253 | ) 254 | svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, freeIP) 255 | svc, err = s.Clientset.Core().Services(svc.ObjectMeta.Namespace).Update(svc) 256 | if err != nil { 257 | glog.Errorf("Unable to update ExternalIPs for service '%v'. Details: %v", 258 | svc.ObjectMeta, err) 259 | } 260 | if setLBIp { 261 | svc.Status.LoadBalancer = v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: freeIP}}} 262 | _, err = s.Clientset.Core().Services(svc.ObjectMeta.Namespace).UpdateStatus(svc) 263 | if err != nil { 264 | glog.Errorf("Error updating service %s status with LoadBalancer IP: %v", svc.Name, err) 265 | } 266 | } 267 | 268 | } 269 | 270 | func (s *ipClaimScheduler) claimWatcher(stop chan struct{}) { 271 | store, controller := cache.NewInformer( 272 | s.claimSource, 273 | &extensions.IpClaim{}, 274 | 10*time.Second, 275 | cache.ResourceEventHandlerFuncs{ 276 | AddFunc: func(obj interface{}) { 277 | claim := obj.(*extensions.IpClaim) 278 | glog.V(3).Infof("IP claim '%v' was created", claim.Metadata.Name) 279 | key, err := cache.MetaNamespaceKeyFunc(claim) 280 | if err != nil { 281 | glog.Errorf("Error getting key for IP claim: %v", err) 282 | } 283 | s.queue.Add(key) 284 | }, 285 | UpdateFunc: func(_, cur interface{}) { 286 | claim := cur.(*extensions.IpClaim) 287 | glog.V(3).Infof("IP claim '%v' was updated. Resource version: %v", 288 | claim.Metadata.Name, claim.Metadata.ResourceVersion) 289 | key, err := cache.MetaNamespaceKeyFunc(claim) 290 | if err != nil { 291 | glog.Errorf("Error getting key for IP claim: %v", err) 292 | } 293 | s.queue.Add(key) 294 | }, 295 | DeleteFunc: func(obj interface{}) { 296 | claim := obj.(*extensions.IpClaim) 297 | glog.V(3).Infof("IP claim '%v' was deleted. Resource version: %v", 298 | claim.Metadata.Name, claim.Metadata.ResourceVersion) 299 | }, 300 | }, 301 | ) 302 | s.claimStore = store 303 | controller.Run(stop) 304 | } 305 | 306 | func (s *ipClaimScheduler) findAliveNodes(ipnodes []extensions.IpNode) (result []*extensions.IpNode) { 307 | s.liveSync.Lock() 308 | s.liveSync.Unlock() 309 | for i := range ipnodes { 310 | node := ipnodes[i] 311 | if _, ok := s.liveIpNodes[node.Metadata.Name]; ok { 312 | result = append(result, &node) 313 | } 314 | } 315 | return result 316 | } 317 | 318 | func (s *ipClaimScheduler) worker() { 319 | glog.V(1).Infof("Starting worker to process IP claims") 320 | for { 321 | key, quit := s.queue.Get() 322 | glog.V(3).Infof("Got IP claim '%v' to process", key) 323 | if quit { 324 | return 325 | } 326 | item, exists, _ := s.claimStore.GetByKey(key.(string)) 327 | if exists { 328 | err := s.processIpClaim(item.(*extensions.IpClaim)) 329 | if err != nil { 330 | glog.Errorf("Error processing IP claim: %v", err) 331 | s.queue.Add(key) 332 | } 333 | } 334 | glog.V(5).Infof("Processing of IP claim '%v' was completed", key) 335 | s.queue.Done(key) 336 | } 337 | } 338 | 339 | func (s *ipClaimScheduler) claimChangeWorker() { 340 | client := s.ExtensionsClientset.IPClaims() 341 | for { 342 | req, quit := s.changeQueue.Get() 343 | glog.V(3).Infof("Got IP claim change request '%v' to process", req) 344 | if quit { 345 | return 346 | } 347 | changeReq := req.(*cache.Delta) 348 | claim := changeReq.Object.(*extensions.IpClaim) 349 | switch changeReq.Type { 350 | case cache.Added: 351 | _, err := client.Create(claim) 352 | if apierrors.IsAlreadyExists(err) { 353 | // Let's add new owner ref to the owner ref list of the existing IP claim 354 | glog.V(3).Infof("IP claim '%v' exists already", claim.Metadata.Name) 355 | existing, err := client.Get(claim.Metadata.Name) 356 | if err != nil { 357 | glog.Errorf("Unable to get IP claim '%v'. Details: %v", claim.Metadata.Name, err) 358 | s.changeQueue.Add(changeReq) 359 | } 360 | newOwnerRef := claim.Metadata.OwnerReferences[0] 361 | existOwnerRefs := existing.Metadata.OwnerReferences 362 | alreadyThere := false 363 | for r := range existOwnerRefs { 364 | if newOwnerRef.UID == existOwnerRefs[r].UID { 365 | glog.V(5).Infof("Service '%v' is referenced in IP claim '%v' already", 366 | newOwnerRef.UID, claim.Metadata.Name) 367 | alreadyThere = true 368 | break 369 | } 370 | } 371 | if !alreadyThere { 372 | existing.Metadata.OwnerReferences = append(existOwnerRefs, newOwnerRef) 373 | s.addClaimChangeRequest(existing, cache.Updated) 374 | glog.V(3).Infof("IP claim '%v' is to be updated with reference to service '%v'", 375 | claim.Metadata.Name, newOwnerRef.UID) 376 | } 377 | } else if err == nil { 378 | glog.V(3).Infof("IP claim '%v' was created", claim.Metadata.Name) 379 | } else { 380 | glog.Errorf("Unable to create IP claim '%v'. Details: %v", claim.Metadata.Name, err) 381 | } 382 | case cache.Updated: 383 | _, err := client.Update(claim) 384 | if err == nil { 385 | glog.V(3).Infof("IP claim '%v' was updated with node '%v'. Resource version: %v", 386 | claim.Metadata.Name, claim.Spec.NodeName, claim.Metadata.ResourceVersion) 387 | } else { 388 | glog.Errorf("Unable to update IP claim '%v'. Details: %v", claim.Metadata.Name, err) 389 | } 390 | case cache.Deleted: 391 | glog.V(5).Infof("claim %s will be deleted", claim.Metadata.Name) 392 | err := client.Delete(claim.Metadata.Name, &metav1.DeleteOptions{}) 393 | if err != nil { 394 | glog.Errorf("Unable to delete IP claim '%v'. Details: %v", claim.Metadata.Name, err) 395 | } 396 | } 397 | glog.V(3).Infof("Processing of IP claim '%v' change request was completed", claim.Metadata.Name) 398 | s.changeQueue.Done(req) 399 | } 400 | } 401 | 402 | func (s *ipClaimScheduler) addClaimChangeRequest(claim *extensions.IpClaim, change cache.DeltaType) { 403 | req := &cache.Delta{ 404 | Object: claim, 405 | Type: change, 406 | } 407 | s.changeQueue.Add(req) 408 | } 409 | 410 | func (s *ipClaimScheduler) processOldService(svc *v1.Service) { 411 | refs := map[string]struct{}{} 412 | svcList := s.serviceStore.List() 413 | for i := range svcList { 414 | svc := svcList[i].(*v1.Service) 415 | for _, ip := range svc.Spec.ExternalIPs { 416 | refs[ip] = struct{}{} 417 | } 418 | } 419 | 420 | pools := s.getIPClaimPoolList() 421 | for _, ip := range svc.Spec.ExternalIPs { 422 | if _, ok := refs[ip]; !ok { 423 | s.deleteIPClaimAndAllocation(ip, pools) 424 | } 425 | } 426 | } 427 | 428 | func (s *ipClaimScheduler) getIPClaimByIP(ip string, pools *extensions.IpClaimPoolList) (*extensions.IpClaim, error) { 429 | mask := "" 430 | if p := poolByAllocatedIP(ip, pools); p != nil { 431 | mask = strings.Split(p.Spec.CIDR, "/")[1] 432 | } else { 433 | mask = s.DefaultMask 434 | } 435 | ipParts := strings.Split(ip, ".") 436 | key := strings.Join([]string{strings.Join(ipParts, "-"), mask}, "-") 437 | return s.ExtensionsClientset.IPClaims().Get(key) 438 | } 439 | 440 | func (s *ipClaimScheduler) getIPClaimPoolList() *extensions.IpClaimPoolList { 441 | pools, err := s.ExtensionsClientset.IPClaimPools().List(metav1.ListOptions{}) 442 | if err != nil { 443 | glog.Errorf("Error retrieving list of IP pools. Details: %v", err) 444 | } 445 | return pools 446 | } 447 | 448 | func (s *ipClaimScheduler) deleteIPClaimAndAllocation(ip string, pools *extensions.IpClaimPoolList) { 449 | glog.V(5).Infof("adding delete request for a clai with ip %s", ip) 450 | if p := poolByAllocatedIP(ip, pools); p != nil { 451 | s.addClaimChangeRequest(makeIPClaim(ip, strings.Split(p.Spec.CIDR, "/")[1], nil), cache.Deleted) 452 | delete(p.Spec.Allocated, ip) 453 | 454 | glog.V(2).Infof("Try to update IP pool with object %v", p) 455 | _, err := s.ExtensionsClientset.IPClaimPools().Update(p) 456 | if err != nil { 457 | glog.Errorf("Unable to update IP pool '%v'. Details: %v", p.Metadata.Name, err) 458 | } 459 | } else { 460 | s.addClaimChangeRequest(makeIPClaim(ip, s.DefaultMask, nil), cache.Deleted) 461 | } 462 | } 463 | 464 | // returns list of owner references that are relevant at the moment 465 | func (s *ipClaimScheduler) ownersAlive(claim *extensions.IpClaim) []metav1.OwnerReference { 466 | // only services can be the claim owners for now 467 | owners := []metav1.OwnerReference{} 468 | for _, owner := range claim.Metadata.OwnerReferences { 469 | _, exists, err := s.serviceStore.GetByKey(string(owner.UID)) 470 | if err != nil { 471 | glog.Errorf("Checking claim '%v' owners: error getting service '%v' from cache: %v", claim.Metadata.Name, owner.UID, err) 472 | } 473 | if !exists { 474 | glog.V(5).Infof("Checking claim '%v' owners: service '%v' is not in cache", claim.Metadata.Name, owner.UID) 475 | ns_name := strings.Split(string(owner.UID), "/") 476 | // "an empty namespace may not be set when a resource name is provided" error is thrown when 477 | // calling Services.Get w/o a namespace 478 | if len(ns_name) == 2 { 479 | _, err = s.Clientset.Core().Services(ns_name[0]).Get(ns_name[1], metav1.GetOptions{}) 480 | } else { 481 | glog.Errorf("Checking claim '%v' owners: cannot get namespace for service '%v'", claim.Metadata.Name, owner.UID) 482 | } 483 | if apierrors.IsNotFound(err) { 484 | glog.V(5).Infof("Checking claim '%v' owners: service '%v' does not exist", claim.Metadata.Name, owner.UID) 485 | continue 486 | } 487 | if err != nil { 488 | glog.Errorf("Checking claim '%v' owners: service '%v' get error: %v", claim.Metadata.Name, owner.UID, err) 489 | } 490 | } 491 | owners = append(owners, owner) 492 | } 493 | glog.V(5).Infof("Checking claim '%v' owners: %v", claim.Metadata.Name, owners) 494 | return owners 495 | } 496 | 497 | func (s *ipClaimScheduler) processIpClaim(claim *extensions.IpClaim) error { 498 | glog.V(5).Infof("verifying owners for a claim %s/%s", claim.Metadata.Namespace, claim.Metadata.Name) 499 | ownersAlive := s.ownersAlive(claim) 500 | if len(ownersAlive) == 0 { 501 | glog.V(5).Infof("owners of a claim %s/%s do not exist", claim.Metadata.Namespace, claim.Metadata.Name) 502 | // all owner links are irrelevant 503 | pools := s.getIPClaimPoolList() 504 | s.deleteIPClaimAndAllocation(strings.Split(claim.Spec.Cidr, "/")[0], pools) 505 | return nil 506 | } else if len(ownersAlive) < len(claim.Metadata.OwnerReferences) { 507 | // some owner links are irrelevant 508 | claim.Metadata.OwnerReferences = ownersAlive 509 | s.addClaimChangeRequest(claim, cache.Updated) 510 | } 511 | glog.V(5).Infof("owners of a claim %s are alive: %v", claim.Metadata.Name, ownersAlive) 512 | if claim.Spec.NodeName != "" && s.isLive(claim.Spec.NodeName) { 513 | return nil 514 | } 515 | ipnodes, err := s.ExtensionsClientset.IPNodes().List(metav1.ListOptions{}) 516 | if err != nil { 517 | return err 518 | } 519 | // this needs to be queued and requeued in case of node absence 520 | if len(ipnodes.Items) == 0 { 521 | return fmt.Errorf("No nodes") 522 | } 523 | liveNodes := s.findAliveNodes(ipnodes.Items) 524 | if len(liveNodes) == 0 { 525 | return fmt.Errorf("No live nodes") 526 | } 527 | ipnode := s.getNode(liveNodes) 528 | claim.Metadata.SetLabels(map[string]string{"ipnode": ipnode.Metadata.Name}) 529 | claim.Spec.NodeName = ipnode.Metadata.Name 530 | glog.V(3).Infof("Scheduling IP claim '%v' on a node '%v'", 531 | claim.Metadata.Name, claim.Spec.NodeName) 532 | s.addClaimChangeRequest(claim, cache.Updated) 533 | return nil 534 | } 535 | 536 | func (s *ipClaimScheduler) monitorIPNodes(stop chan struct{}, ticker <-chan time.Time) { 537 | for { 538 | select { 539 | case <-stop: 540 | return 541 | case <-ticker: 542 | ipnodes, err := s.ExtensionsClientset.IPNodes().List(metav1.ListOptions{}) 543 | if err != nil { 544 | glog.Errorf("Error getting IP nodes: %v", err) 545 | } 546 | 547 | for _, ipnode := range ipnodes.Items { 548 | name := ipnode.Metadata.Name 549 | version := s.observedGeneration[name] 550 | curVersion := ipnode.Revision 551 | if version < curVersion { 552 | s.observedGeneration[name] = curVersion 553 | s.liveSync.Lock() 554 | glog.V(3).Infof("IP node '%v' is alive. Versions: %v - %v", 555 | name, version, curVersion) 556 | s.liveIpNodes[name] = struct{}{} 557 | s.liveSync.Unlock() 558 | } else { 559 | s.liveSync.Lock() 560 | glog.V(3).Infof("IP node '%v' is dead. Versions: %v - %v", 561 | name, version, curVersion) 562 | delete(s.liveIpNodes, name) 563 | s.liveSync.Unlock() 564 | labelSelector := labels.Set(map[string]string{"ipnode": name}) 565 | ipclaims, err := s.ExtensionsClientset.IPClaims().List( 566 | metav1.ListOptions{ 567 | LabelSelector: labelSelector.String(), 568 | }, 569 | ) 570 | if err != nil { 571 | glog.Errorf("Error fetching list of IP claims: %v", err) 572 | break 573 | } 574 | for _, ipclaim := range ipclaims.Items { 575 | // TODO don't update - send to queue for rescheduling instead 576 | glog.Infof("Sending IP claim '%v' for rescheduling. CIDR '%v', previous node '%v'", 577 | ipclaim.Metadata.Name, ipclaim.Spec.Cidr, ipclaim.Spec.NodeName) 578 | key, err := cache.MetaNamespaceKeyFunc(&ipclaim) 579 | if err != nil { 580 | glog.Errorf("Error getting key for IP claim: %v", err) 581 | } else { 582 | s.queue.Add(key) 583 | } 584 | } 585 | } 586 | } 587 | } 588 | } 589 | } 590 | 591 | func (s *ipClaimScheduler) isLive(name string) bool { 592 | s.liveSync.Lock() 593 | defer s.liveSync.Unlock() 594 | _, ok := s.liveIpNodes[name] 595 | return ok 596 | } 597 | 598 | func checkAnnotation(svc *v1.Service) bool { 599 | if svc.ObjectMeta.Annotations != nil { 600 | val, exists := svc.ObjectMeta.Annotations[AutoExternalAnnotationKey] 601 | if exists { 602 | glog.V(5).Infof( 603 | "Auto-allocation annotation (key '%v') is provided for service '%v' with value '%v'", 604 | AutoExternalAnnotationKey, svc.ObjectMeta.Name, val, 605 | ) 606 | if val == AutoExternalAnnotationValue { 607 | return true 608 | } 609 | glog.Warning("Only value '%v' is processed for annotation key '%v'", 610 | AutoExternalAnnotationValue, AutoExternalAnnotationKey) 611 | } 612 | } 613 | return false 614 | } 615 | 616 | func updatePoolAllocation(ext extensions.ExtensionsClientset, pool *extensions.IpClaimPool, ip, claimName string) error { 617 | if pool.Spec.Allocated != nil { 618 | pool.Spec.Allocated[ip] = claimName 619 | } else { 620 | pool.Spec.Allocated = map[string]string{ip: claimName} 621 | } 622 | 623 | glog.V(2).Infof("Update IP pool with object %v", pool) 624 | _, err := ext.IPClaimPools().Update(pool) 625 | return err 626 | } 627 | 628 | func makeIPClaim(ip, mask string, svc *v1.Service) *extensions.IpClaim { 629 | ipParts := strings.Split(ip, ".") 630 | key := strings.Join([]string{strings.Join(ipParts, "-"), mask}, "-") 631 | cidr := strings.Join([]string{ip, mask}, "/") 632 | 633 | glog.V(2).Infof("Creating IP claim '%v'", key) 634 | meta := metav1.ObjectMeta{Name: key} 635 | if svc != nil { 636 | svc_key, err := cache.MetaNamespaceKeyFunc(svc) 637 | if err != nil { 638 | return nil 639 | } 640 | ctrl := false 641 | ownerRef := metav1.OwnerReference{APIVersion: "v1", Kind: "ServiceReference", Name: svc.Name, UID: types.UID(svc_key), Controller: &ctrl} 642 | meta.OwnerReferences = []metav1.OwnerReference{ownerRef} 643 | } 644 | 645 | ipclaim := &extensions.IpClaim{ 646 | Metadata: meta, 647 | Spec: extensions.IpClaimSpec{Cidr: cidr}, 648 | } 649 | return ipclaim 650 | } 651 | --------------------------------------------------------------------------------