├── .gitignore ├── .gitmodules ├── .travis.yml ├── Docker ├── Dockerfile ├── README.md └── entrypoint.sh ├── LICENSE ├── Makefile ├── README.md ├── build.sh ├── deployments ├── README.md ├── kubernetes │ ├── README.md │ ├── Vagrantfile │ ├── deployments │ │ ├── busybox.yaml │ │ └── ubuntu.yaml │ ├── example.conf │ └── k8s-bootstrap.sh └── single-host │ ├── README.md │ ├── Vagrantfile │ └── configs │ └── etcd.service ├── example ├── example-centalip-cluster.conf ├── example-centalip-node.conf ├── example.conf └── multus │ ├── ovs-network-cluster.yaml │ └── ovs-network-node.yaml ├── ipam └── centralip │ ├── README.md │ ├── backend │ ├── central-ip.go │ ├── central-ip_test.go │ ├── cluster │ │ ├── cluster.go │ │ └── cluster_test.go │ ├── node │ │ ├── node.go │ │ └── node_test.go │ └── utils │ │ ├── types.go │ │ ├── utils.go │ │ └── utils_test.go │ └── main.go ├── ovs ├── backend │ ├── disk │ │ ├── backend.go │ │ └── backend_test.go │ └── store.go ├── main.go ├── ovs_switch.go ├── ovs_switch_test.go ├── ovs_test.go ├── utils.go └── utils_test.go └── vendor └── vendor.json /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | bin/ 3 | gopath/ 4 | .vagrant 5 | vendor/* 6 | !vendor/vendor.json 7 | *.log -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deployments/kubespray"] 2 | path = deployments/kubespray 3 | url = https://github.com/hwchiu/kubespray 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: true 3 | go: 4 | - 1.9.x 5 | 6 | notifications: 7 | email: 8 | recipients: 9 | - hwchiu@linkernetworks.com 10 | - cwlin@linkernetworks.com 11 | on_success: change 12 | before_install: 13 | - sudo apt-get install -y git build-essential openvswitch-switch 14 | - go get -u github.com/kardianos/govendor 15 | - go get -u github.com/pierrre/gotestcover 16 | - docker run -d -p 2379:2379 --name etcd quay.io/coreos/etcd:v3.2 /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 17 | install: 18 | - govendor sync 19 | script: 20 | - sudo -E env PATH=$PATH TEST_ETCD=1 gotestcover -coverprofile=coverage.txt -covermode=atomic ./... 21 | after_success: 22 | - bash <(curl -s https://codecov.io/bash) 23 | -------------------------------------------------------------------------------- /Docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | ADD bin/ /tmp/bin 4 | ADD example/ /tmp/conf 5 | ADD example/multus /tmp/conf 6 | ADD Docker/entrypoint.sh / 7 | 8 | ENTRYPOINT ["/entrypoint.sh"] 9 | -------------------------------------------------------------------------------- /Docker/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | In order to use the **kubespray** to automatically deploy the kubenetes cluster with ovs-cni. 3 | We will modify the **kubespray** project and modify the ansible playbook. 4 | In the network config option, we add new config 'ovs' and it will run this dockerfile to copy the pre-build 5 | binary and config into each kubernetes cluster. 6 | 7 | ## Kubespray 8 | You can find the custom **kubespray** project [here](https://github.com/hwchiu/kubespray). 9 | In that project, just type **vagrant up** and you can use the kubernetes cluster with ovs-cni now. 10 | 11 | 12 | ## How to Build docker image. 13 | Use the **Make docker-build** to build the docker image. 14 | 15 | ## How to run the docker image 16 | 1. First, you should create two directory. 17 | ``` 18 | mkdir -p /tmp/test/bin 19 | mkdir -p /tmp/test/conf 20 | ``` 21 | 2. run the docker image and mount those two directory 22 | ``` 23 | sudo docker run -it --rm -v /tmp/test/bin:/opt/cni/bin -v /tmp/test/conf:/etc/cni/net.d/ hwchiu/ovs-cni central-cluster 1.2.3.4 24 | ``` 25 | 3. The entrypoing supports two arguments, IPAM type and etcd IP. 26 | - IPAM: **central-cluster**/**central-node** 27 | 4. After executing the container, you will see two binaries in **/tmp/test/bin** and a ovs.conf in **/tmp/test/conf** 28 | -------------------------------------------------------------------------------- /Docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | handleConfig() 4 | { 5 | confType=$1 6 | etcdIP=$2 7 | targetDir=$3 8 | 9 | srcFile="" 10 | dstFile="" 11 | ##The multus need the yaml, otherwise the cni config. 12 | case $confType in 13 | central-node) 14 | srcFile="/tmp/conf/example-centalip-node.conf" 15 | dstFile="ovs.conf" 16 | ;; 17 | central-cluster) 18 | srcFile="/tmp/conf/example-centalip-cluster.conf" 19 | dstFile="ovs.conf" 20 | ;; 21 | multus-node) 22 | srcFile="/tmp/conf/ovs-network-node.yaml" 23 | dstFile="ovs-network.yaml" 24 | ;; 25 | multus-cluster) 26 | srcFile="/tmp/conf/ovs-network-cluster.yaml" 27 | dstFile="ovs-network.yaml" 28 | ;; 29 | *) 30 | ;; 31 | esac 32 | sed -i "s/127.0.0.1/$etcdIP/g" $srcFile 33 | cp $srcFile $3/$dstFile 34 | } 35 | 36 | confType="centralip-cluster" 37 | etcdIP="127.0.0.1" 38 | 39 | while getopts b:t:i:c: option 40 | do 41 | case "${option}" 42 | in 43 | b) echo "Copy CNI Binary to "${OPTARG} 44 | cp /tmp/bin/* ${OPTARG} 45 | ;; 46 | t) 47 | confType=`echo ${OPTARG} | cut -d ':' -f1` 48 | ;; 49 | i) 50 | etcdIP=${OPTARG} 51 | ;; 52 | c) 53 | echo "Copty CNI Conf to "${OPTARG} 54 | handleConfig $confType $etcdIP ${OPTARG} 55 | ;; 56 | esac 57 | done 58 | 59 | exit 60 | 61 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: bin 2 | 3 | bin: clean 4 | sh build.sh 5 | 6 | clean: 7 | rm -rf bin 8 | docker-build: bin 9 | sudo docker build -t hwchiu/ovs-cni:latest -f Docker/Dockerfile . 10 | 11 | docker-push: docker-build 12 | sudo docker push hwchiu/ovs-cni:latest 13 | test: 14 | go get -u github.com/pierrre/gotestcover 15 | sudo -E env PATH=$$PATH TEST_ETCD=1 gotestcover -coverprofile=coverage.txt -covermode=atomic ./... 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ovs-cni 2 | 3 | [![Build Status](https://api.travis-ci.org/John-Lin/ovs-cni.svg?branch=master)](https://travis-ci.org/John-Lin/ovs-cni) 4 | [![codecov](https://codecov.io/gh/John-Lin/ovs-cni/branch/master/graph/badge.svg)](https://codecov.io/gh/John-Lin/ovs-cni) 5 | 6 | ## Introduction 7 | 8 | OVS-CNI is a totally Open vSwitch CNI plugin, it create the Open vSwitch and use veth to connect the OpenvSwitch and container. 9 | 10 | ovs-cni supports the following options in its config and you can see the example config `example/example.conf` to learn how to use it. 11 | 12 | 1. controller IP 13 | 14 | ``` 15 | controller:10.245.1.5:6653 16 | ``` 17 | 18 | 2. target vxlan IP 19 | 20 | ``` 21 | vtepIPs: [ 22 | "10.245.2.2", 23 | "10.245.2.3" 24 | ] 25 | ``` 26 | 27 | 3. bridge name 28 | 29 | ``` 30 | ovsBridge: "br0" 31 | ``` 32 | 33 | 4. Act as a gateway for pods. 34 | 35 | ``` 36 | isDefaultGateway: true 37 | ``` 38 | 39 | 5. Support SNAT for pods traffic. 40 | 41 | ``` 42 | ipMasq: true 43 | ``` 44 | 45 | 6. IPAM support 46 | 47 | ovs-cni support basic IPAM type such as host-local, you can see `example/example.conf` to see how config it. 48 | Besides, ovs-cni provide a new IPAM plugin central-ip, which use the `ETCD` to perform centralized IP assignment/management and you can go to `ipam/centralip` directory to see more usage about it. 49 | 50 | ## Usage 51 | 52 | If you are familiar with CNI plugin and know how to use it. you can refer to the following instruction to use it. 53 | Otherwise you can go to `deployment` directory to learn how to use it. 54 | 55 | ### Building ovs-cni 56 | 57 | Because ovs-cni used the package management tool called `govendor`, we have to install the govendor first. 58 | 59 | ``` 60 | $ go get -u github.com/kardianos/govendor 61 | ``` 62 | 63 | We use `govendor` to download all dependencies 64 | 65 | ``` 66 | $ cd ~/go/src/github.com/John-Lin/ovs-cni 67 | $ govendor sync 68 | ``` 69 | 70 | build the ovs-cni binary. 71 | 72 | ``` 73 | $ ./build.sh 74 | ``` 75 | and the binary will come out in the `/bin` directory and you can find `ovs` and `centralip`. 76 | The `ovs` is the main CNI plugin and the `centralip` is the CNI plugin for different IPAM usage. 77 | If you want to use `ETCD` to centralizaed manage the IP address, you should also copy the `centralip` binary to the CNI directory and modify the config to use it. 78 | 79 | ``` 80 | $ sudo ip netns add ns1 81 | 82 | $ sudo CNI_COMMAND=ADD CNI_CONTAINERID=ns1 CNI_NETNS=/var/run/netns/ns1 CNI_IFNAME=eth2 CNI_PATH=`pwd` ./ovs : --discovery-token-ca-cert-hash sha256: 159 | ``` 160 | 161 | Use `kubeadm join` on host2 to join the machine. 162 | 163 | ### After join number of machines by running kubeadm join 164 | 165 | Restart kubelet on **BOTH virtual machines** to activate the `KUBELET_EXTRA_ARGS` has configured before: 166 | 167 | ``` 168 | $ sudo systemctl daemon-reload 169 | $ sudo systemctl restart kubelet 170 | ``` 171 | 172 | ## Use ovs-cni as a default network interface 173 | 174 | ### Configuring ovs-cni network plugin 175 | 176 | Modify the configuration to meet your requirements. Check the configuration files in the `example` and copy one into the default `--cni-conf-dir` path which is in `/etc/cni/net.d` and rename it as `ovs.conf` 177 | 178 | For example, if you want to use the simplest configuration, you can type following command to copy the configuration file 179 | 180 | #### Host-Local 181 | ``` 182 | $ cd ~/go/src/github.com/John-Lin/ovs-cni 183 | $ cp example/example.conf /etc/cni/net.d/ovs.conf 184 | ``` 185 | **Note**: the `vtepIPs`, `rangeStart`, `rangeEnd` and `gateway` could be different on each host. 186 | 187 | #### CentralIP 188 | If you want to use the centralizaed IP management, you can use the following command 189 | ``` 190 | $ cd ~/go/src/github.com/John-Lin/ovs-cni 191 | $ cp example/example-centalip-node.conf /etc/cni/net.d/ovs.conf 192 | ``` 193 | After that, you should modify the `etcdURL` option to your k8s master IP address. 194 | Besides, you should also modify the etcd manifests to allow etcd server running on public (from 127.0.0.1 to 0.0.0.0) in kuberneter master node (This could cause security issue) 195 | 196 | ```shell 197 | sudo vim /etc/kubernetes/manifests/etcd.yaml 198 | ``` 199 | 200 | ```yaml 201 | ... 202 | spec: 203 | containers: 204 | - command: 205 | - etcd 206 | - --listen-client-urls=http://0.0.0.0:2379 207 | - --advertise-client-urls=http://0.0.0.0:2379 208 | - --data-dir=/var/lib/etcd 209 | ... 210 | ... 211 | ``` 212 | 213 | Restart kubelet on master node 214 | 215 | ``` 216 | $ sudo systemctl daemon-reload 217 | $ sudo systemctl restart kubelet 218 | ``` 219 | 220 | ### Master Isolation 221 | 222 | By default, your cluster will not schedule pods on the master for security reasons. If you want to be able to schedule pods on the master, e.g. for a single-machine Kubernetes cluster for development, run: 223 | 224 | ``` 225 | $ kubectl taint nodes --all node-role.kubernetes.io/master- 226 | ``` 227 | 228 | ### Apply the deployment on Kubernetes cluster 229 | 230 | ``` 231 | $ cd ~/go/src/github.com/John-Lin/ovs-cni 232 | $ kubectl apply -f ./kubernetes/deployments/busybox.yaml 233 | $ kubectl get pod -o wide 234 | NAME READY STATUS RESTARTS AGE IP NODE 235 | busybox-deployment-6b8c55d957-6wjcl 1/1 Running 11 1d 10.244.2.11 host2 236 | busybox-deployment-6b8c55d957-pxm6c 1/1 Running 11 1d 10.244.1.12 host1 237 | ``` 238 | 239 | ## Extend the multiple network interfaces with OVS CNI 240 | 241 | ### Build and install Multus plugin 242 | 243 | This sould be do on ALL kubernetes nodes 244 | 245 | ### Building multus 246 | ```shell 247 | git cloen https://github.com/Intel-Corp/multus-cni.git 248 | cd multus-cni 249 | ./build 250 | ``` 251 | 252 | ### Installation 253 | Copy the binary from `/bin/multus` to `/opt/cni/bin/` and make sure the `ovs` binary is inside the directory 254 | 255 | Create Multus CNI configuration file `/etc/cni/net.d/multus-cni.conf` with below content in minions. Use only the absolute path to point to the kubeconfig file (it may change depending upon your cluster env) and make sure all CNI binary files are in `/opt/cni/bin` dir 256 | 257 | ``` 258 | { 259 | "name": "minion-cni-network", 260 | "type": "multus", 261 | "kubeconfig": "/home/ubuntu/.kube/config", 262 | "delegates": [{ 263 | "type": "ovs", 264 | "masterplugin": true 265 | }] 266 | } 267 | ``` 268 | 269 | You might need to copy kubeconfig file from the kubernetes master node `/home/ubuntu/.kube/config` to all minion nodes. This could allow Multus works. 270 | 271 | ### Create a Custom Resource Definition (CRD) based Network objects 272 | 273 | Create a Third party resource `crdnetwork.yaml` for the network object 274 | 275 | ```yaml 276 | apiVersion: apiextensions.k8s.io/v1beta1 277 | kind: CustomResourceDefinition 278 | metadata: 279 | # name must match the spec fields below, and be in the form: . 280 | name: networks.kubernetes.com 281 | spec: 282 | # group name to use for REST API: /apis// 283 | group: kubernetes.com 284 | # version name to use for REST API: /apis// 285 | version: v1 286 | # either Namespaced or Cluster 287 | scope: Namespaced 288 | names: 289 | # plural name to be used in the URL: /apis/// 290 | plural: networks 291 | # singular name to be used as an alias on the CLI and for display 292 | singular: network 293 | # kind is normally the CamelCased singular type. Your resource manifests use this. 294 | kind: Network 295 | # shortNames allow shorter string to match your resource on the CLI 296 | shortNames: 297 | - net 298 | ``` 299 | 300 | kubectl create command for the Custom Resource Definition 301 | 302 | ```sh 303 | # kubectl create -f ./crdnetwork.yaml 304 | customresourcedefinition "network.kubernetes.com" created 305 | ``` 306 | 307 | kubectl get command to check the Network CRD creation 308 | 309 | ```sh 310 | # kubectl get CustomResourceDefinition 311 | # kubectl get crd 312 | NAME KIND 313 | networks.kubernetes.com CustomResourceDefinition.v1beta1.apiextensions.k8s.io 314 | ``` 315 | 316 | Save the below following YAML to ovs-network.yaml 317 | 318 | ```yaml 319 | apiVersion: "kubernetes.com/v1" 320 | kind: Network 321 | metadata: 322 | name: ovs-net 323 | plugin: ovs 324 | args: '[ 325 | { 326 | "name": "myovsnet", 327 | "type": "ovs", 328 | "ovsBridge":"br0", 329 | "isDefaultGateway": true, 330 | "ipMasq": true, 331 | "ipam":{ 332 | "type":"centralip", 333 | "network":"10.245.0.0/16", 334 | "subnetLen": 24, 335 | "subnetMin": "10.245.5.0", 336 | "subnetMax": "10.245.50.0", 337 | "etcdURL": "192.168.0.107:2379" 338 | } 339 | } 340 | ]' 341 | ``` 342 | 343 | With ipam type `centralip` should setup a ETCD server. By default it should be set to kubernetes master server IP. 344 | 345 | Create the ovs network object 346 | 347 | ```shell 348 | # kubectl create -f ovs-network.yaml 349 | network "ovs-net" created 350 | ``` 351 | 352 | Check the network object 353 | 354 | ```shell 355 | # kubectl get network 356 | # kubectl get net 357 | ``` 358 | 359 | Next, modify the etcd manifests to allow etcd server running on public (from 127.0.0.1 to 0.0.0.0) in kuberneter master node (This could cause security issue) 360 | 361 | ```shell 362 | sudo vim /etc/kubernetes/manifests/etcd.yaml 363 | ``` 364 | 365 | ```yaml 366 | ... 367 | spec: 368 | containers: 369 | - command: 370 | - etcd 371 | - --listen-client-urls=http://0.0.0.0:2379 372 | - --advertise-client-urls=http://0.0.0.0:2379 373 | - --data-dir=/var/lib/etcd 374 | ... 375 | ... 376 | ``` 377 | 378 | Restart kubelet on master node 379 | 380 | ``` 381 | $ sudo systemctl daemon-reload 382 | $ sudo systemctl restart kubelet 383 | ``` 384 | 385 | Save the below following YAML to flannel-network.yaml 386 | 387 | ``` 388 | apiVersion: "kubernetes.com/v1" 389 | kind: Network 390 | metadata: 391 | name: flannel-networkobj 392 | plugin: flannel 393 | args: '[ 394 | { 395 | "delegate": { 396 | "isDefaultGateway": true 397 | } 398 | } 399 | ]' 400 | ``` 401 | 402 | Create the flannel network object 403 | 404 | ``` 405 | # kubectl create -f flannel-network.yaml 406 | network "flannel-networkobj" created 407 | ``` 408 | 409 | ``` 410 | # kubectl get network 411 | NAME KIND ARGS PLUGIN 412 | flannel-networkobj Network.v1.kubernetes.com [ { "delegate": { "isDefaultGateway": true } } ] flannel 413 | ``` 414 | 415 | ### Configuring Pod to use the CRD Network objects 416 | 417 | Save the below following YAML to pod-multi-network.yaml. In this case `ovs-net` network object act as the primary network. 418 | 419 | ```yaml 420 | # cat pod-multi-network.yaml 421 | apiVersion: v1 422 | kind: Pod 423 | metadata: 424 | name: multus-multi-net-poc 425 | annotations: 426 | networks: '[ 427 | { "name": "ovs-net" }, 428 | { "name": "flannel-conf" } 429 | ]' 430 | spec: # specification of the pod's contents 431 | containers: 432 | - name: multus-multi-net-poc 433 | image: "busybox" 434 | command: ["top"] 435 | stdin: true 436 | tty: true 437 | ``` 438 | 439 | For setting up `flannel-conf` please see 440 | 441 | https://github.com/coreos/flannel/blob/4c057be1f97a38960436834f144b0bd824d7f76e/README.md#multi-network-mode-experimental 442 | 443 | https://github.com/coreos/flannel/blob/master/Documentation/running.md 444 | 445 | Create Multiple network based pod from the master node 446 | 447 | ```shell 448 | # kubectl create -f ./pod-multi-network.yaml 449 | pod "multus-multi-net-poc" created 450 | ``` 451 | 452 | 453 | ### References 454 | 455 | - https://github.com/Intel-Corp/multus-cni 456 | - https://kubernetes.io/docs/concepts/api-extension/custom-resources/#custom-resources 457 | -------------------------------------------------------------------------------- /deployments/kubernetes/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | cluster = { 5 | "host1" => { :cpus => 1, :mem => 1024, :ip => "192.168.2.3" }, 6 | "host2" => { :cpus => 1, :mem => 1024, :ip => "192.168.2.4" }, 7 | } 8 | 9 | Vagrant.configure("2") do |config| 10 | # with privileged true 11 | config.vm.provision :shell, path: "k8s-bootstrap.sh" 12 | 13 | config.vm.provision "shell", privileged: false, inline: <<-SHELL 14 | set -e -x -u 15 | sudo apt-get update 16 | sudo apt-get install -y vim git build-essential openvswitch-switch tcpdump 17 | # Listen on the given TCP port for a connection 18 | sudo ovs-appctl -t ovsdb-server ovsdb-server/add-remote ptcp:6640 19 | # Install Golang 20 | wget --quiet https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz 21 | sudo tar -zxf go1.9.1.linux-amd64.tar.gz -C /usr/local/ 22 | echo 'export GOROOT=/usr/local/go' >> /home/ubuntu/.bashrc 23 | echo 'export GOPATH=$HOME/go' >> /home/ubuntu/.bashrc 24 | echo 'export PATH=$PATH:$GOROOT/bin:$GOPATH/bin' >> /home/ubuntu/.bashrc 25 | export GOROOT=/usr/local/go 26 | export GOPATH=$HOME/go 27 | export PATH=$PATH:$GOROOT/bin:$GOPATH/bin 28 | mkdir -p /home/ubuntu/go/src 29 | rm -rf /home/ubuntu/go1.9.1.linux-amd64.tar.gz 30 | # Download CNI and CNI plugins binaries 31 | wget --quiet https://github.com/containernetworking/cni/releases/download/v0.6.0/cni-amd64-v0.6.0.tgz 32 | wget --quiet https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz 33 | sudo mkdir -p /opt/cni/bin 34 | sudo mkdir -p /etc/cni/net.d 35 | sudo tar -zxf cni-amd64-v0.6.0.tgz -C /opt/cni/bin 36 | sudo tar -zxf cni-plugins-amd64-v0.6.0.tgz -C /opt/cni/bin 37 | rm -rf /home/ubuntu/cni-plugins-amd64-v0.6.0.tgz /home/ubuntu/cni-amd64-v0.6.0.tgz 38 | # Download ovs CNI source and build the binary 39 | git clone https://github.com/John-Lin/ovs-cni /home/ubuntu/go/src/github.com/John-Lin/ovs-cni 40 | go get -u github.com/kardianos/govendor 41 | cd ~/go/src/github.com/John-Lin/ovs-cni 42 | govendor sync 43 | ./build.sh 44 | # Download multus CNI sourc and build the binary 45 | git clone https://github.com/Intel-Corp/multus-cni.git /home/ubuntu/go/src/github.com/Intel-Corp/multus-cni 46 | cd ~/go/src/github.com/Intel-Corp/multus-cni 47 | ./build 48 | # Install CNI plugins 49 | sudo cp ~/go/src/github.com/John-Lin/ovs-cni/bin/ovs /opt/cni/bin 50 | sudo cp ~/go/src/github.com/John-Lin/ovs-cni/bin/centralip /opt/cni/bin 51 | sudo cp ~/go/src/github.com/Intel-Corp/multus-cni/bin/multus /opt/cni/bin 52 | SHELL 53 | 54 | cluster.each_with_index do |(hostname, info), index| 55 | config.vm.define hostname do |cfg| 56 | cfg.vm.box = "ubuntu/xenial64" 57 | cfg.vm.hostname = hostname 58 | cfg.vm.network "private_network", ip: info[:ip] 59 | cfg.vm.provider :virtualbox do |v| 60 | v.customize ["modifyvm", :id, "--cpus", info[:cpus]] 61 | v.customize ["modifyvm", :id, "--memory", info[:mem]] 62 | v.customize ['modifyvm', :id, '--nicpromisc1', 'allow-all'] 63 | v.customize ["modifyvm", :id, "--name", hostname] 64 | end # end provider 65 | end # end config 66 | end # end cluster 67 | end 68 | -------------------------------------------------------------------------------- /deployments/kubernetes/deployments/busybox.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: busybox-deployment 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: busybox 9 | replicas: 5 10 | template: 11 | metadata: 12 | labels: 13 | app: busybox 14 | spec: 15 | containers: 16 | - name: busybox 17 | image: busybox 18 | command: 19 | - sleep 20 | - "3600" 21 | -------------------------------------------------------------------------------- /deployments/kubernetes/deployments/ubuntu.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: ubuntu-deployment-demo 5 | spec: 6 | replicas: 3 7 | template: 8 | metadata: 9 | labels: 10 | demo: deployment 11 | version: v1 12 | spec: 13 | containers: 14 | - name: ubuntu 15 | image: ubuntu:14.04 16 | command: 17 | - sleep 18 | - "3600" -------------------------------------------------------------------------------- /deployments/kubernetes/example.conf: -------------------------------------------------------------------------------- 1 | { 2 | "name":"mynet", 3 | "cniVersion":"0.3.1", 4 | "type":"ovs", 5 | "ovsBridge":"br0", 6 | "vtepIPs":[ 7 | "10.0.0.3" 8 | ], 9 | "isDefaultGateway": true, 10 | "ipMasq": true, 11 | "ipam":{ 12 | "type":"centralip", 13 | "network":"10.245.0.0/16", 14 | "subnetLen": 24, 15 | "subnetMin": "10.245.5.0", 16 | "subnetMax": "10.245.50.0", 17 | "etcdURL": "127.0.0.1:2379" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /deployments/kubernetes/k8s-bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | apt install ebtables ethtool 3 | 4 | apt-get update 5 | apt-get install -y docker.io 6 | 7 | apt-get update && apt-get install -y apt-transport-https 8 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - 9 | cat </etc/apt/sources.list.d/kubernetes.list 10 | deb http://apt.kubernetes.io/ kubernetes-xenial main 11 | EOF 12 | apt-get update 13 | apt-get install -y kubelet kubeadm kubectl -------------------------------------------------------------------------------- /deployments/single-host/README.md: -------------------------------------------------------------------------------- 1 | # Single Host Vagrant 2 | This document intends to give a instruction about how to try ovs-cni with docker/namespace in vagrant. 3 | 4 | ## Environment 5 | You should install vagrant in your system and make sure everything goes well. 6 | 7 | ## Setup ovs-CNI 8 | - Type `vagrant up` to init a virtual machine. 9 | - Use ssh to connect vagrant VM via `vagrant ssh`. 10 | - Type following command to build the `ovs-cni` binary and move it to CNI directory. 11 | ``` 12 | $ cd ~/go/src/github.com/John-Lin/ovs-cni 13 | $ sudo cp ./bin/ovs /opt/cni/bin 14 | ``` 15 | - We need to provide a CNI config example for `ovs-cni`, and you can use build-in config from example directory. Use following command to copy the config to `~/cni` directory. 16 | 17 | ``` 18 | $ sudo cp examples/example.conf ~/cni/ 19 | ``` 20 | 21 | ## Create NS 22 | In this vagrant environment, we don't install docker related services but you can use `namespace(ns)` to test `ovs-cni`. 23 | Type following command to create a namespace named ns1 24 | 25 | ``` 26 | $ sudo ip netns add ns1 27 | ``` 28 | 29 | ## Start CNI 30 | We have setup `ovs-cni` environement and testing namespace(ns1), we can use following command to inform `ovs-cni` to add a network for the namespace. 31 | 32 | ``` 33 | $cd ~/cni 34 | $ sudo CNI_COMMAND=ADD CNI_CONTAINERID=ns1 CNI_NETNS=/var/run/netns/ns1 CNI_IFNAME=eth2 CNI_PATH=`pwd` ./ovs > /home/ubuntu/.bashrc 17 | echo 'export GOPATH=$HOME/go' >> /home/ubuntu/.bashrc 18 | echo 'export PATH=$PATH:$GOROOT/bin:$GOPATH/bin' >> /home/ubuntu/.bashrc 19 | export GOROOT=/usr/local/go 20 | export GOPATH=$HOME/go 21 | export PATH=$PATH:$GOROOT/bin:$GOPATH/bin 22 | mkdir -p /home/ubuntu/go/src 23 | rm -rf /home/ubuntu/go1.9.1.linux-amd64.tar.gz 24 | # Download CNI and CNI plugins binaries 25 | wget --quiet https://github.com/containernetworking/cni/releases/download/v0.6.0/cni-amd64-v0.6.0.tgz 26 | wget --quiet https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz 27 | sudo mkdir -p /opt/cni/bin 28 | sudo mkdir -p /etc/cni/net.d 29 | sudo tar -zxf cni-amd64-v0.6.0.tgz -C /opt/cni/bin 30 | sudo tar -zxf cni-plugins-amd64-v0.6.0.tgz -C /opt/cni/bin 31 | rm -rf /home/ubuntu/cni-plugins-amd64-v0.6.0.tgz /home/ubuntu/cni-amd64-v0.6.0.tgz 32 | # Download ovs CNI source 33 | git clone https://github.com/John-Lin/ovs-cni go/src/github.com/John-Lin/ovs-cni 34 | go get -u github.com/kardianos/govendor 35 | cd ~/go/src/github.com/John-Lin/ovs-cni 36 | govendor sync 37 | # build the ovs-cni binary 38 | ./build.sh 39 | sudo cp ~/go/src/github.com/John-Lin/ovs-cni/bin/ovs /opt/cni/bin 40 | sudo cp ~/go/src/github.com/John-Lin/ovs-cni/bin/centralip /opt/cni/bin 41 | wget --quiet https://github.com/coreos/etcd/releases/download/v3.0.7/etcd-v3.0.7-linux-amd64.tar.gz 42 | sudo tar xzvf etcd-v3.0.7-linux-amd64.tar.gz -C /opt/ 43 | sudo mv /opt/etcd-v3.0.7-linux-amd64 /opt/etcd 44 | sudo cp /tmp/etcd.service /etc/systemd/system/etcd.service 45 | sudo sudo systemctl enable etcd 46 | sudo sudo systemctl start etcd 47 | SHELL 48 | 49 | config.vm.provider :virtualbox do |v| 50 | v.customize ["modifyvm", :id, "--cpus", 4] 51 | # enable this when you want to have more memory 52 | # v.customize ["modifyvm", :id, "--memory", 4096] 53 | v.customize ['modifyvm', :id, '--nicpromisc1', 'allow-all'] 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /deployments/single-host/configs/etcd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Etcd Server 3 | Documentation=https://github.com/coreos/etcd 4 | After=network.target 5 | 6 | [Service] 7 | User=root 8 | Type=simple 9 | ExecStart=/opt/etcd/etcd 10 | Restart=on-failure 11 | RestartSec=10s 12 | LimitNOFILE=40000 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /example/example-centalip-cluster.conf: -------------------------------------------------------------------------------- 1 | { 2 | "name":"mynet", 3 | "cniVersion":"0.3.1", 4 | "type":"ovs", 5 | "ovsBridge":"br0", 6 | "ipam":{ 7 | "type":"centralip", 8 | "ipType": "cluster", 9 | "network":"10.245.0.0/16", 10 | "etcdURL": "https://127.0.0.1:2379", 11 | "etcdCertFile": "/etc/ovs/certs/cert.crt", 12 | "etcdKeyFile": "/etc/ovs/certs/key.pem", 13 | "etcdTrustedCAFileFile": "/etc/ovs/certs/ca_cert.crt" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /example/example-centalip-node.conf: -------------------------------------------------------------------------------- 1 | { 2 | "name":"mynet", 3 | "cniVersion":"0.3.1", 4 | "type":"ovs", 5 | "ovsBridge":"br0", 6 | "vtepIPs":[ 7 | "10.0.0.3" 8 | ], 9 | "isDefaultGateway": true, 10 | "ipMasq": true, 11 | "ipam":{ 12 | "type":"centralip", 13 | "ipType": "node", 14 | "network":"10.245.0.0/16", 15 | "subnetLen": 24, 16 | "subnetMin": "10.245.5.0", 17 | "subnetMax": "10.245.50.0", 18 | "etcdURL": "127.0.0.1:2379" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /example/example.conf: -------------------------------------------------------------------------------- 1 | { 2 | "name":"mynet", 3 | "cniVersion":"0.3.1", 4 | "type":"ovs", 5 | "ovsBridge":"br0", 6 | "vtepIPs":[ 7 | "10.245.2.2", 8 | "10.245.2.3" 9 | ], 10 | "isDefaultGateway": true, 11 | "ipMasq": true, 12 | "ipam":{ 13 | "type":"host-local", 14 | "subnet":"10.244.0.0/16", 15 | "rangeStart":"10.244.1.10", 16 | "rangeEnd":"10.244.1.150", 17 | "routes":[ 18 | { 19 | "dst":"0.0.0.0/0" 20 | } 21 | ], 22 | "gateway":"10.244.1.1" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /example/multus/ovs-network-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "kubernetes.com/v1" 2 | kind: Network 3 | metadata: 4 | name: ovs-net 5 | plugin: ovs 6 | args: '[ 7 | { 8 | "name": "myovsnet", 9 | "type": "ovs", 10 | "ovsBridge":"br0", 11 | "ipam":{ 12 | "type":"centralip", 13 | "ipType": "cluster", 14 | "network":"10.245.0.0/16", 15 | "etcdURL": "127.0.0.1:2379" 16 | } 17 | } 18 | ]' 19 | -------------------------------------------------------------------------------- /example/multus/ovs-network-node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "kubernetes.com/v1" 2 | kind: Network 3 | metadata: 4 | name: ovs-net 5 | plugin: ovs 6 | args: '[ 7 | { 8 | "name": "myovsnet", 9 | "type": "ovs", 10 | "ovsBridge":"br0", 11 | "isDefaultGateway": true, 12 | "ipMasq": true, 13 | "ipam":{ 14 | "type":"centralip", 15 | "network":"10.245.0.0/16", 16 | "subnetLen": 24, 17 | "subnetMin": "10.245.5.0", 18 | "subnetMax": "10.245.50.0", 19 | "etcdURL": "127.0.0.1:2379" 20 | } 21 | } 22 | ]' 23 | -------------------------------------------------------------------------------- /ipam/centralip/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | The centralip IPAM plugin use the etcd-v3 to handle all ip management. 3 | We provide two mode to decide how to dispatch the ip address. 4 | `Node` mode and `clster` mode. 5 | In the cluster mode, we even don't provide the gateway address for each node and we hope the SDN controller should handle this, such as ONOS. 6 | 7 | ## config 8 | The config of centralip like below. 9 | ``` 10 | "ipam":{ 11 | "type":"centralip", 12 | "ipType": "node", 13 | "network":"10.245.0.0/16", 14 | "subnetLen": 24, 15 | "subnetMin": "10.245.5.0", 16 | "subnetMax": "10.245.50.0", 17 | "etcdURL": "127.0.0.1:2379" 18 | } 19 | ``` 20 | ### ipType 21 | We have two backends now, `node` and `cluster`. 22 | 23 | In the `node` mode, You need to specify all the following options and it will assign different ip subnet to each node. 24 | 25 | In the ohter hand, the `cluster` mode, you only set the `network` and `etcdURL` options. 26 | The `cluster` will assign the IP address to all nodes in the same subnet. 27 | In this mode, we won't provide the gateway address for you, so don't set the `IsDefaultGatway` option in your CNI configuration. 28 | You can see two example configs in the `../../examples` 29 | 30 | ### network 31 | This field indicate the whole network subnet you want to use. 32 | 33 | ### subnetLen 34 | This length and `network` will decide the CIDR for each node. 35 | For example, the centralip will dispatch the subnet `10.245.1.0/24`, `10.245.2.0/24`.... `10.245.255.0/24` 36 | 37 | ### subnetMin/subnetMax 38 | Those two fields is used to indicate the range of subnets you want to dispatch for each node. 39 | 40 | ### etcdURL 41 | The ip address of etcd-v3 server. 42 | If you want to connect to etcd-v3 servier with TLS, you should also indicate the 43 | location of three files, including `Certificate`, `Key` and `TrustedCA`. 44 | And you should use the following key to describe your location to CNI. 45 | ```bash 46 | "etcdURL": "https://127.0.0.1:2379", 47 | "etcdCertFile": "/etc/ovs/certs/cert.crt", 48 | "etcdKeyFile": "/etc/ovs/certs/key.pem", 49 | "etcdTrustedCAFileFile": "/etc/ovs/certs/ca_cert.crt" 50 | ``` 51 | -------------------------------------------------------------------------------- /ipam/centralip/backend/central-ip.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package centralip 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/cluster" 21 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/node" 22 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/utils" 23 | "github.com/containernetworking/cni/pkg/skel" 24 | "os" 25 | ) 26 | 27 | type CentralNet struct { 28 | Name string `json:"name"` 29 | CNIVersion string `json:"cniVersion"` 30 | IPM *utils.IPMConfig `json:"ipam"` 31 | } 32 | 33 | func GenerateCentralIPM(args *skel.CmdArgs) (utils.CentralIPM, error, string) { 34 | n := &CentralNet{} 35 | if err := json.Unmarshal(args.StdinData, n); err != nil { 36 | return nil, fmt.Errorf("failed to load netconf: %v", err), "" 37 | } 38 | 39 | switch n.IPM.IPType { 40 | case "node": 41 | hostname, _ := os.Hostname() 42 | node, err := node.New(args.ContainerID, hostname, n.IPM) 43 | return node, err, n.CNIVersion 44 | case "cluster": 45 | node, err := cluster.New(args.ContainerID, n.IPM) 46 | return node, err, n.CNIVersion 47 | default: 48 | return nil, fmt.Errorf("Unsupport IPM type %s", n.IPM.Type), "" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ipam/centralip/backend/central-ip_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package centralip 16 | 17 | import ( 18 | "github.com/containernetworking/cni/pkg/skel" 19 | "github.com/stretchr/testify/assert" 20 | "os" 21 | "testing" 22 | ) 23 | 24 | var validNodeData = skel.CmdArgs{ 25 | StdinData: []byte(` 26 | { 27 | "name":"mynet", 28 | "cniVersion":"0.3.1", 29 | "ipam":{ 30 | "type":"central", 31 | "ipType": "node", 32 | "network":"10.245.0.0/16", 33 | "subnetLen": 24, 34 | "subnetMin": "10.245.5.0", 35 | "subnetMax": "10.245.6.0", 36 | "etcdURL": "127.0.0.1:2379" 37 | } 38 | } 39 | `), 40 | } 41 | 42 | var InvalidData = skel.CmdArgs{ 43 | StdinData: []byte(` 44 | { 45 | "name":"mynet", 46 | "cniVersion":"0.3.1", 47 | "ipam":{ 48 | "type":"central", 49 | "network":"10.245.0.0/16", 50 | "subnetLen": 24, 51 | "subnetMin": "10.245.5.0", 52 | "subnetMax": "10.245.6.0", 53 | "etcdURL": "127.0.0.1:2379" 54 | } 55 | } 56 | `), 57 | } 58 | 59 | var validClusterData = skel.CmdArgs{ 60 | StdinData: []byte(` 61 | { 62 | "name":"mynet", 63 | "cniVersion":"0.3.1", 64 | "ipam":{ 65 | "type":"central", 66 | "ipType": "cluster", 67 | "network":"10.245.0.0/16", 68 | "subnetLen": 24, 69 | "subnetMin": "10.245.5.0", 70 | "subnetMax": "10.245.6.0", 71 | "etcdURL": "127.0.0.1:2379" 72 | } 73 | 74 | } 75 | `), 76 | } 77 | 78 | func TestGenerateCentralIPM(t *testing.T) { 79 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 80 | t.SkipNow() 81 | return 82 | } 83 | t.Run("Node instance", func(t *testing.T) { 84 | n, err, version := GenerateCentralIPM(&validNodeData) 85 | assert.NoError(t, err) 86 | assert.NotNil(t, n) 87 | assert.Equal(t, version, "0.3.1") 88 | }) 89 | t.Run("Cluster instance", func(t *testing.T) { 90 | n, err, version := GenerateCentralIPM(&validClusterData) 91 | assert.NoError(t, err) 92 | assert.NotNil(t, n) 93 | assert.Equal(t, version, "0.3.1") 94 | }) 95 | } 96 | 97 | func TestGenerateInvalidCentralIPM(t *testing.T) { 98 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 99 | t.SkipNow() 100 | return 101 | } 102 | n, err, _ := GenerateCentralIPM(&InvalidData) 103 | assert.Error(t, err) 104 | assert.Nil(t, n) 105 | } 106 | -------------------------------------------------------------------------------- /ipam/centralip/backend/cluster/cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cluster 16 | 17 | import ( 18 | "fmt" 19 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/utils" 20 | "github.com/coreos/etcd/clientv3" 21 | "math/rand" 22 | "net" 23 | ) 24 | 25 | type NodeIPM struct { 26 | cli *clientv3.Client 27 | podname string 28 | subnet *net.IPNet 29 | config *utils.IPMConfig 30 | } 31 | 32 | const clusterPrefix string = utils.ETCDPrefix + "cluster/" 33 | 34 | func New(podName string, config *utils.IPMConfig) (*NodeIPM, error) { 35 | node := &NodeIPM{} 36 | node.config = config 37 | var err error 38 | 39 | node.podname = podName 40 | node.cli, err = utils.ConnectETCD(config) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | _, node.subnet, err = net.ParseCIDR(config.Network) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return node, nil 50 | } 51 | 52 | func (node *NodeIPM) GetGateway() (string, error) { 53 | return "", nil 54 | } 55 | 56 | func (node *NodeIPM) GetAvailableIP() (string, *net.IPNet, error) { 57 | ipnet := &net.IPNet{} 58 | if node.subnet == nil { 59 | return "", ipnet, fmt.Errorf("You should init IPM first") 60 | } 61 | 62 | usedIPPrefix := clusterPrefix + "used/" 63 | 64 | length, _ := node.subnet.Mask.Size() 65 | ipRange := utils.PowTwo(32 - (length)) 66 | //Since the first IP is gateway, we should skip it 67 | 68 | //change to random a ip (must not be gateway) and try to check the etcd: 69 | start := utils.GetNextIP(node.subnet) 70 | 71 | //If ip Range = 256, our target it 2~255 72 | //rand.Intn( range - 2 ) will return 0<=n<254, 73 | //+2 will cause 2<=n<256 74 | retryTimes := 20 75 | var availableIP string 76 | for i := 0; i < retryTimes; i++ { 77 | tryIP := utils.GetIPByInt(start, uint32(rand.Intn(int(ipRange-2))+1)) 78 | 79 | ipUsedToPod, err := utils.GetKeyValuesWithPrefix(node.cli,usedIPPrefix) 80 | if err != nil { 81 | return "", ipnet, err 82 | } 83 | 84 | //check. 85 | if _, ok := ipUsedToPod[usedIPPrefix+tryIP.String()]; !ok { 86 | availableIP = tryIP.String() 87 | utils.PutValue(node.cli, usedIPPrefix+tryIP.String(), node.podname) 88 | break 89 | } 90 | } 91 | 92 | var err error 93 | //We need to generate a net.IPnet object which contains the IP and Mask. 94 | //We use ParseCIDR to create the net.IPnet object and assign IP back to it. 95 | cidr := fmt.Sprintf("%s/%d", availableIP, length) 96 | var ip net.IP 97 | ip, ipnet, err = net.ParseCIDR(cidr) 98 | if err != nil { 99 | return "", ipnet, err 100 | } 101 | 102 | ipnet.IP = ip 103 | return availableIP, ipnet, nil 104 | } 105 | 106 | func (node *NodeIPM) Delete() error { 107 | //get all used ip address and try to matches it id. 108 | usedIPPrefix := clusterPrefix + "used/" 109 | ipUsedToPod, err := utils.GetKeyValuesWithPrefix(node.cli,usedIPPrefix) 110 | if err != nil { 111 | return err 112 | } 113 | 114 | for k, v := range ipUsedToPod { 115 | fmt.Println(k, v) 116 | if v == node.podname { 117 | err := utils.DeleteKey(node.cli, k) 118 | return err 119 | } 120 | } 121 | return fmt.Errorf("There aren't any infomation about pod %s", node.podname) 122 | } 123 | -------------------------------------------------------------------------------- /ipam/centralip/backend/cluster/cluster_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cluster 16 | 17 | import ( 18 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/utils" 19 | "github.com/stretchr/testify/assert" 20 | "os" 21 | "testing" 22 | "time" 23 | ) 24 | 25 | var node *NodeIPM 26 | var err error 27 | 28 | var validData = utils.IPMConfig{ 29 | Network: "10.123.0.0/16", 30 | ETCDURL: "127.0.0.1:2379", 31 | } 32 | 33 | func TestNewNode(t *testing.T) { 34 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 35 | t.SkipNow() 36 | return 37 | } 38 | node, err = New("pod1", &validData) 39 | assert.NoError(t, err) 40 | assert.NotNil(t, node) 41 | assert.Equal(t, node.config.ETCDURL, "127.0.0.1:2379") 42 | } 43 | 44 | func TestGetGateway(t *testing.T) { 45 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 46 | t.SkipNow() 47 | return 48 | } 49 | gwIP, err := node.GetGateway() 50 | assert.NoError(t, err) 51 | assert.Equal(t, "", gwIP) 52 | } 53 | 54 | func TestGetAvailableIP(t *testing.T) { 55 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 56 | t.SkipNow() 57 | return 58 | } 59 | t.Run("First IP", func(t *testing.T) { 60 | ip, ipNet, err := node.GetAvailableIP() 61 | assert.NoError(t, err) 62 | assert.NotEqual(t, "10.123.0.1/16", ipNet.String()) 63 | assert.NotEqual(t, "10.123.0.1", ip) 64 | }) 65 | time.Sleep(1 * time.Second) 66 | t.Run("Second IP", func(t *testing.T) { 67 | ip, ipNet, err := node.GetAvailableIP() 68 | assert.NoError(t, err) 69 | assert.NotEqual(t, "10.123.0.1/16", ipNet.String()) 70 | assert.NotEqual(t, "10.123.0.1", ip) 71 | }) 72 | time.Sleep(1 * time.Second) 73 | t.Run("remove first IP", func(t *testing.T) { 74 | err := node.Delete() 75 | assert.NoError(t, err) 76 | }) 77 | time.Sleep(1 * time.Second) 78 | t.Run("Fetch IP again", func(t *testing.T) { 79 | ip, ipNet, err := node.GetAvailableIP() 80 | assert.NoError(t, err) 81 | assert.NotEqual(t, "10.123.0.1/16", ipNet.String()) 82 | assert.NotEqual(t, "10.123.0.1", ip) 83 | }) 84 | } 85 | 86 | func TestSecondHost(t *testing.T) { 87 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 88 | t.SkipNow() 89 | return 90 | } 91 | node2, err := New("pod2", &validData) 92 | assert.NoError(t, err) 93 | 94 | gwIP, err := node2.GetGateway() 95 | assert.NoError(t, err) 96 | assert.Equal(t, "", gwIP) 97 | ip, ipNet, err := node2.GetAvailableIP() 98 | assert.NoError(t, err) 99 | assert.NotEqual(t, "10.123.0.1/16", ipNet.String()) 100 | assert.NotEqual(t, "10.123.0.1", ip) 101 | 102 | } 103 | 104 | func TestGenerateCentralIPMInvalid(t *testing.T) { 105 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 106 | t.SkipNow() 107 | return 108 | } 109 | t.Run("invalid etcd", func(t *testing.T) { 110 | var InvalidETCD = utils.IPMConfig{ 111 | Network: "10.123.0.0/16", 112 | ETCDURL: "127.0.0.1:23792", 113 | } 114 | var err error 115 | node, err = New("pod1", &InvalidETCD) 116 | assert.Error(t, err) 117 | assert.Nil(t, node) 118 | }) 119 | t.Run("invalid network", func(t *testing.T) { 120 | var InvalidNetwork = utils.IPMConfig{ 121 | Network: "10.23.0.0/16ds", 122 | ETCDURL: "127.0.0.1:2379", 123 | } 124 | 125 | var err error 126 | node, err = New("pod1", &InvalidNetwork) 127 | assert.Error(t, err) 128 | assert.Nil(t, node) 129 | }) 130 | } 131 | -------------------------------------------------------------------------------- /ipam/centralip/backend/node/node.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package node 16 | 17 | import ( 18 | "fmt" 19 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/utils" 20 | "github.com/coreos/etcd/clientv3" 21 | "math/rand" 22 | "net" 23 | ) 24 | 25 | type NodeIPM struct { 26 | cli *clientv3.Client 27 | hostname string 28 | podname string 29 | subnet *net.IPNet 30 | config *utils.IPMConfig 31 | } 32 | 33 | const nodePrefix string = utils.ETCDPrefix + "node/" 34 | const subnetPrefix string = nodePrefix + "subnets/" 35 | 36 | func New(podName, hostname string, config *utils.IPMConfig) (*NodeIPM, error) { 37 | node := &NodeIPM{} 38 | node.config = config 39 | var err error 40 | 41 | node.hostname = hostname 42 | node.podname = podName 43 | 44 | node.cli, err = utils.ConnectETCD(config) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | err = node.registerNode() 50 | if err != nil { 51 | return nil, err 52 | } 53 | return node, nil 54 | } 55 | 56 | func (node *NodeIPM) checkNodeIsRegisted() error { 57 | 58 | keyValues, err := utils.GetKeyValuesWithPrefix(node.cli,nodePrefix + node.hostname) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | if 0 == len(keyValues) { 64 | return nil 65 | } 66 | 67 | _, node.subnet, err = net.ParseCIDR(keyValues[nodePrefix+node.hostname]) 68 | return err 69 | } 70 | 71 | func (node *NodeIPM) registerSubnet() error { 72 | //Convert the subnet to int. for example. 73 | //string(10.16.7.0) -> net.IP(10.16.7.0) -> int(168822528) 74 | ipnet := net.ParseIP(node.config.SubnetMin) 75 | ipStart, err := utils.IpToInt(ipnet) 76 | if err != nil { 77 | fmt.Println(node) 78 | return err 79 | } 80 | 81 | //Since the subnet len is 24, we need to add 2^(32-24) for each subnet. 82 | //(168822528 + 2^8) == 10.16.8.0 83 | //(168822528 + 2* 2 ^8 ) == 10.16.9.0 84 | ipNextSubnet := utils.PowTwo(32 - node.config.SubnetLen) 85 | ipEnd := net.ParseIP(node.config.SubnetMax) 86 | 87 | nextSubnet := utils.IntToIP(ipStart) 88 | 89 | nodeToSubnets, err :=utils.GetKeyValuesWithPrefix(node.cli,subnetPrefix) 90 | 91 | if err != nil { 92 | return err 93 | } 94 | 95 | for i := 1; ; i++ { 96 | cidr := fmt.Sprintf("%s%s/%d", subnetPrefix, nextSubnet.String(), node.config.SubnetLen) 97 | 98 | if _, ok := nodeToSubnets[cidr]; !ok { 99 | break 100 | } 101 | if ipEnd.String() == nextSubnet.String() { 102 | return fmt.Errorf("No available subnet for registering") 103 | } 104 | nextSubnet = utils.IntToIP(ipStart + ipNextSubnet*uint32(i)) 105 | } 106 | 107 | subnet := &net.IPNet{IP: nextSubnet, Mask: net.CIDRMask(node.config.SubnetLen, 32)} 108 | node.subnet = subnet 109 | 110 | //store the $nodePrefix/hostname -> subnet 111 | err = utils.PutValue(node.cli,nodePrefix+node.hostname, subnet.String()) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | //store the $nodePrefix/subnets/$subnet -> hostname for fast lookup for existing subnet 117 | err = utils.PutValue(node.cli,subnetPrefix+subnet.String(), node.hostname) 118 | return err 119 | } 120 | 121 | func (node *NodeIPM) registerNode() error { 122 | //Check Node Exist 123 | err := node.checkNodeIsRegisted() 124 | if err != nil { 125 | return err 126 | } 127 | 128 | if node.subnet == nil { 129 | err := node.registerSubnet() 130 | if err != nil { 131 | return err 132 | } 133 | } 134 | return nil 135 | } 136 | 137 | func (node *NodeIPM) GetGateway() (string, error) { 138 | if node.subnet == nil { 139 | return "", fmt.Errorf("You should init IPM first") 140 | } 141 | 142 | gwPrefix := nodePrefix + node.hostname + "/gateway" 143 | nodeValues, err := utils.GetKeyValuesWithPrefix(node.cli,gwPrefix) 144 | if err != nil { 145 | return "", err 146 | } 147 | 148 | var gwIP string 149 | if len(nodeValues) == 0 { 150 | gwIP = utils.GetNextIP(node.subnet).String() 151 | utils.PutValue(node.cli,gwPrefix, gwIP) 152 | } else { 153 | gwIP = nodeValues[gwPrefix] 154 | } 155 | return gwIP, nil 156 | } 157 | 158 | func (node *NodeIPM) GetAvailableIP() (string, *net.IPNet, error) { 159 | ipnet := &net.IPNet{} 160 | if node.subnet == nil { 161 | return "", ipnet, fmt.Errorf("You should init IPM first") 162 | } 163 | 164 | usedIPPrefix := nodePrefix + node.hostname + "/used/" 165 | ipRange := utils.PowTwo(32 - (node.config.SubnetLen)) 166 | 167 | //change to random a ip (must not be gateway) and try to check the etcd: 168 | start := utils.GetNextIP(node.subnet) 169 | 170 | //If ip Range = 256, our target it 2~255 171 | //rand.Intn( range - 2 ) will return 0<=n<254, 172 | //+2 will cause 2<=n<256 173 | retryTimes := 20 174 | var availableIP string 175 | for i := 0; i < retryTimes; i++ { 176 | tryIP := utils.GetIPByInt(start, uint32(rand.Intn(int(ipRange-2))+1)) 177 | 178 | ipUsedToPod, err := utils.GetKeyValuesWithPrefix(node.cli,usedIPPrefix) 179 | if err != nil { 180 | return "", ipnet, err 181 | } 182 | 183 | //check. 184 | if _, ok := ipUsedToPod[usedIPPrefix+tryIP.String()]; !ok { 185 | availableIP = tryIP.String() 186 | utils.PutValue(node.cli,usedIPPrefix+tryIP.String(), node.podname) 187 | break 188 | } 189 | } 190 | 191 | var err error 192 | //We need to generate a net.IPnet object which contains the IP and Mask. 193 | //We use ParseCIDR to create the net.IPnet object and assign IP back to it. 194 | cidr := fmt.Sprintf("%s/%d", availableIP, node.config.SubnetLen) 195 | var ip net.IP 196 | ip, ipnet, err = net.ParseCIDR(cidr) 197 | if err != nil { 198 | return "", ipnet, err 199 | } 200 | 201 | ipnet.IP = ip 202 | return availableIP, ipnet, nil 203 | } 204 | 205 | func (node *NodeIPM) Delete() error { 206 | //get all used ip address and try to matches it id. 207 | usedIPPrefix := nodePrefix + node.hostname + "/used/" 208 | ipUsedToPod, err := utils.GetKeyValuesWithPrefix(node.cli,usedIPPrefix) 209 | if err != nil { 210 | return err 211 | } 212 | 213 | for k, v := range ipUsedToPod { 214 | if v == node.podname { 215 | err := utils.DeleteKey(node.cli, k) 216 | return err 217 | } 218 | } 219 | return fmt.Errorf("There aren't any infomation about %s", node.hostname) 220 | } 221 | -------------------------------------------------------------------------------- /ipam/centralip/backend/node/node_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package node 16 | 17 | import ( 18 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend/utils" 19 | "github.com/stretchr/testify/assert" 20 | "os" 21 | "testing" 22 | "time" 23 | ) 24 | 25 | var node *NodeIPM 26 | var err error 27 | 28 | var validData = utils.IPMConfig{ 29 | Network: "10.123.0.0/16", 30 | SubnetLen: 24, 31 | SubnetMin: "10.123.5.0", 32 | SubnetMax: "10.123.6.0", 33 | ETCDURL: "127.0.0.1:2379", 34 | } 35 | 36 | func TestNewNode(t *testing.T) { 37 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 38 | t.SkipNow() 39 | return 40 | } 41 | node, err = New("pod1", "host1", &validData) 42 | assert.NoError(t, err) 43 | assert.NotNil(t, node) 44 | assert.Equal(t, node.config.ETCDURL, "127.0.0.1:2379") 45 | } 46 | 47 | func TestGetGateway(t *testing.T) { 48 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 49 | t.SkipNow() 50 | return 51 | } 52 | gwIP, err := node.GetGateway() 53 | assert.NoError(t, err) 54 | assert.Equal(t, "10.123.5.1", gwIP) 55 | gwIP, err = node.GetGateway() 56 | assert.NoError(t, err) 57 | assert.Equal(t, "10.123.5.1", gwIP) 58 | } 59 | 60 | func TestGetAvailableIP(t *testing.T) { 61 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 62 | t.SkipNow() 63 | return 64 | } 65 | t.Run("First IP", func(t *testing.T) { 66 | ip, ipNet, err := node.GetAvailableIP() 67 | assert.NoError(t, err) 68 | assert.NotEqual(t, "10.123.5.1/24", ipNet.String()) 69 | assert.NotEqual(t, "10.123.5.1", ip) 70 | }) 71 | time.Sleep(1 * time.Second) 72 | t.Run("Second IP", func(t *testing.T) { 73 | ip, ipNet, err := node.GetAvailableIP() 74 | assert.NoError(t, err) 75 | assert.NotEqual(t, "10.123.5.1/24", ipNet.String()) 76 | assert.NotEqual(t, "10.123.5.1", ip) 77 | }) 78 | time.Sleep(1 * time.Second) 79 | t.Run("remove first IP", func(t *testing.T) { 80 | err := node.Delete() 81 | assert.NoError(t, err) 82 | }) 83 | time.Sleep(1 * time.Second) 84 | t.Run("Fetch IP again", func(t *testing.T) { 85 | ip, ipNet, err := node.GetAvailableIP() 86 | assert.NoError(t, err) 87 | assert.NotEqual(t, "10.123.5.1/24", ipNet.String()) 88 | assert.NotEqual(t, "10.123.5.1", ip) 89 | }) 90 | } 91 | 92 | func TestSecondHost(t *testing.T) { 93 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 94 | t.SkipNow() 95 | return 96 | } 97 | node2, err := New("pod1", "host2", &validData) 98 | assert.NoError(t, err) 99 | 100 | gwIP, err := node2.GetGateway() 101 | assert.NoError(t, err) 102 | assert.Equal(t, "10.123.6.1", gwIP) 103 | ip, ipNet, err := node2.GetAvailableIP() 104 | assert.NoError(t, err) 105 | assert.NotEqual(t, "10.123.6.1/24", ipNet.String()) 106 | assert.NotEqual(t, "10.123.6.1", ip) 107 | 108 | } 109 | 110 | func TestGenerateCentralIPMInvalid(t *testing.T) { 111 | if _, defined := os.LookupEnv("TEST_ETCD"); !defined { 112 | t.SkipNow() 113 | return 114 | } 115 | var InvalidData = utils.IPMConfig{ 116 | Network: "10.123.0.0/16", 117 | SubnetLen: 24, 118 | SubnetMin: "10.123.5.0", 119 | SubnetMax: "10.123.6.0", 120 | ETCDURL: "127.0.0.1:23792", 121 | } 122 | 123 | t.Run("invalid etcd", func(t *testing.T) { 124 | var err error 125 | node, err = New("pod1", "host1", &InvalidData) 126 | assert.Error(t, err) 127 | assert.Nil(t, node) 128 | }) 129 | t.Run("no available subnet", func(t *testing.T) { 130 | var err error 131 | node, err = New("pod1", "host3", &validData) 132 | assert.Error(t, err) 133 | assert.Nil(t, node) 134 | }) 135 | 136 | } 137 | -------------------------------------------------------------------------------- /ipam/centralip/backend/utils/types.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | type IPMConfig struct { 8 | Type string `json:"type"` 9 | IPType string `json:"ipType"` 10 | Network string `json:"network"` 11 | SubnetLen int `json:"subnetLen"` 12 | SubnetMin string `json:"subnetMin"` 13 | SubnetMax string `json:"subnetMax"` 14 | ETCDURL string `json:"etcdURL"` 15 | ETCDCertFile string `json:"etcdCertFile"` 16 | ETCDKeyFile string `json:"etcdKeyFile"` 17 | ETCDTrustedCAFileFile string `json:"etcdTrustedCAFileFile"` 18 | } 19 | 20 | type CentralIPM interface { 21 | GetGateway() (string, error) 22 | GetAvailableIP() (string, *net.IPNet, error) 23 | Delete() error 24 | } 25 | 26 | const ETCDPrefix string = "/ovs-cni/networks/" 27 | -------------------------------------------------------------------------------- /ipam/centralip/backend/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Che Wei, Lin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "context" 19 | "github.com/coreos/etcd/clientv3" 20 | "github.com/coreos/etcd/pkg/transport" 21 | "time" 22 | "encoding/binary" 23 | "fmt" 24 | "github.com/containernetworking/plugins/pkg/ip" 25 | "net" 26 | "strings" 27 | ) 28 | 29 | func PowTwo(times int) uint32 { 30 | if times == 0 { 31 | return uint32(1) 32 | } 33 | 34 | var ans uint32 35 | ans = 1 36 | for i := 0; i < times; i++ { 37 | ans *= 2 38 | } 39 | 40 | return ans 41 | } 42 | 43 | func IpToInt(ip net.IP) (uint32, error) { 44 | if v4 := ip.To4(); v4 != nil { 45 | if len(ip) == 16 { 46 | return binary.BigEndian.Uint32(ip[12:16]), nil 47 | } else { 48 | return binary.BigEndian.Uint32(ip[0:4]), nil 49 | } 50 | } 51 | return 0, fmt.Errorf("IP should be ipv4 %v\n", ip) 52 | } 53 | 54 | func IntToIP(nn uint32) net.IP { 55 | ip := make(net.IP, 4) 56 | binary.BigEndian.PutUint32(ip, nn) 57 | return ip 58 | } 59 | 60 | //We use the first IP as gateway address 61 | func GetNextIP(ipn *net.IPNet) net.IP { 62 | nid := ipn.IP.Mask(ipn.Mask) 63 | return ip.NextIP(nid) 64 | } 65 | 66 | func GetIPByInt(ip net.IP, n uint32) net.IP { 67 | i, _ := IpToInt(ip) 68 | return IntToIP(i + n) 69 | } 70 | 71 | /* 72 | ETCD Related 73 | */ 74 | func connectWithoutTLS(url string) (*clientv3.Client, error) { 75 | cli, err := clientv3.New(clientv3.Config{ 76 | Endpoints: []string{url}, 77 | DialTimeout: 5 * time.Second, 78 | }) 79 | 80 | return cli, err 81 | } 82 | 83 | func connectWithTLS(url, cert, key, trusted string) (*clientv3.Client, error) { 84 | tlsInfo := transport.TLSInfo{ 85 | CertFile: cert, 86 | KeyFile: key, 87 | TrustedCAFile: trusted, 88 | } 89 | 90 | tlsConfig, err := tlsInfo.ClientConfig() 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | cli, err := clientv3.New(clientv3.Config{ 96 | Endpoints: []string{url}, 97 | DialTimeout: 5 * time.Second, 98 | TLS: tlsConfig, 99 | }) 100 | 101 | return cli, err 102 | } 103 | 104 | 105 | func ConnectETCD(config *IPMConfig) (*clientv3.Client, error) { 106 | var cli *clientv3.Client 107 | var err error 108 | if strings.HasPrefix(config.ETCDURL, "https") { 109 | cli, err = connectWithTLS(config.ETCDURL, config.ETCDCertFile, config.ETCDKeyFile, config.ETCDTrustedCAFileFile) 110 | } else { 111 | cli, err = connectWithoutTLS(config.ETCDURL) 112 | } 113 | 114 | return cli,err 115 | } 116 | 117 | 118 | func DeleteKey(cli *clientv3.Client, prefix string) error { 119 | _, err := cli.Delete(context.TODO(), prefix) 120 | return err 121 | } 122 | func PutValue(cli *clientv3.Client,prefix, value string) error { 123 | _, err := cli.Put(context.TODO(), prefix, value) 124 | return err 125 | } 126 | 127 | func GetKeyValuesWithPrefix(cli *clientv3.Client, key string) (map[string]string, error) { 128 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 129 | resp, err := cli.Get(ctx, key, clientv3.WithPrefix()) 130 | cancel() 131 | if err != nil { 132 | return nil, fmt.Errorf("Fetch etcd prefix error:%v", err) 133 | } 134 | 135 | results := make(map[string]string) 136 | for _, ev := range resp.Kvs { 137 | results[string(ev.Key)] = string(ev.Value) 138 | } 139 | 140 | return results, nil 141 | } 142 | -------------------------------------------------------------------------------- /ipam/centralip/backend/utils/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Che Wei, Lin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "github.com/stretchr/testify/assert" 19 | "net" 20 | "testing" 21 | ) 22 | 23 | func TestPowOfTwo(t *testing.T) { 24 | assert.Equal(t, uint32(2), PowTwo(1)) 25 | assert.Equal(t, uint32(1), PowTwo(0)) 26 | assert.Equal(t, uint32(1024), PowTwo(10)) 27 | assert.Equal(t, uint32(2147483648), PowTwo(31)) 28 | } 29 | 30 | func TestIp2Int(t *testing.T) { 31 | v4Input := net.ParseIP("127.0.0.1") 32 | result, err := IpToInt(v4Input) 33 | assert.NoError(t, err) 34 | assert.Equal(t, uint32(2130706433), result) 35 | 36 | v6Input := net.ParseIP("2001:0DB8:02de:0000:0000:0000:0000:0e13") 37 | result, err = IpToInt(v6Input) 38 | assert.Error(t, err) 39 | } 40 | 41 | func TestInt2IP(t *testing.T) { 42 | input := IntToIP(2130706433) 43 | assert.Equal(t, "127.0.0.1", input.String()) 44 | 45 | } 46 | 47 | func TestGetGatewayFromIP(t *testing.T) { 48 | _, input, _ := net.ParseCIDR("192.168.194.0/22") 49 | 50 | gwIP := GetNextIP(input) 51 | assert.Equal(t, gwIP.String(), "192.168.192.1") 52 | } 53 | 54 | func TestGetIpByInt(t *testing.T) { 55 | net, _, _ := net.ParseCIDR("192.168.194.0/22") 56 | 57 | assert.Equal(t, GetIPByInt(net, uint32(20)).String(), "192.168.194.20") 58 | assert.Equal(t, GetIPByInt(net, uint32(0)).String(), "192.168.194.0") 59 | assert.Equal(t, GetIPByInt(net, uint32(50)).String(), "192.168.194.50") 60 | assert.Equal(t, GetIPByInt(net, uint32(200)).String(), "192.168.194.200") 61 | assert.Equal(t, GetIPByInt(net, uint32(300)).String(), "192.168.195.44") 62 | } 63 | -------------------------------------------------------------------------------- /ipam/centralip/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net" 5 | // "strings" 6 | "github.com/John-Lin/ovs-cni/ipam/centralip/backend" 7 | "github.com/containernetworking/cni/pkg/skel" 8 | "github.com/containernetworking/cni/pkg/types" 9 | "github.com/containernetworking/cni/pkg/types/current" 10 | "github.com/containernetworking/cni/pkg/version" 11 | ) 12 | 13 | func main() { 14 | skel.PluginMain(cmdAdd, cmdDel, version.All) 15 | } 16 | 17 | /* 18 | 19 | type CmdArgs struct { 20 | ContainerID string 21 | Netns string 22 | IfName string 23 | Args string 24 | Path string 25 | StdinData []byte 26 | } 27 | 28 | */ 29 | func cmdAdd(args *skel.CmdArgs) error { 30 | n, err, cniversion := centralip.GenerateCentralIPM(args) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | gwIP, err := n.GetGateway() 36 | _, IP, err := n.GetAvailableIP() 37 | if err != nil { 38 | return err 39 | } 40 | 41 | i := net.ParseIP(gwIP) 42 | 43 | version := "4" 44 | if IP.IP.To4() == nil { 45 | version = "6" 46 | } 47 | ipconfig := ¤t.IPConfig{ 48 | Version: version, 49 | Address: *IP, 50 | Gateway: i, 51 | } 52 | 53 | result := ¤t.Result{} 54 | result.IPs = append(result.IPs, ipconfig) 55 | result.Routes = []*types.Route{} 56 | return types.PrintResult(result, cniversion) 57 | } 58 | 59 | func cmdDel(args *skel.CmdArgs) error { 60 | n, err, _ := centralip.GenerateCentralIPM(args) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | err = n.Delete() 66 | if err != nil { 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /ovs/backend/disk/backend.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | const defaultDataDir = "/var/lib/cni/networks" 11 | 12 | // Store is a simple disk-backed store that creates one file per network namespace 13 | // container ID is a given filename. The contents of the file are the interface name for ovs. 14 | type Store struct { 15 | dataDir string 16 | } 17 | 18 | func New(network, dataDir string) (*Store, error) { 19 | if dataDir == "" { 20 | dataDir = defaultDataDir 21 | } 22 | dir := filepath.Join(dataDir, network) 23 | if err := os.MkdirAll(dir, 0755); err != nil { 24 | return nil, err 25 | } 26 | 27 | return &Store{dir}, nil 28 | } 29 | 30 | func (s *Store) Reserve(id, ovsIfaceName string) (bool, error) { 31 | fname := strings.TrimSpace(id) 32 | fpath := filepath.Join(s.dataDir, fname) 33 | 34 | f, err := os.OpenFile(fpath, os.O_RDWR|os.O_EXCL|os.O_CREATE, 0644) 35 | if os.IsExist(err) { 36 | return false, nil 37 | } 38 | if err != nil { 39 | return false, err 40 | } 41 | 42 | if _, err := f.WriteString(ovsIfaceName); err != nil { 43 | f.Close() 44 | os.Remove(f.Name()) 45 | return false, err 46 | } 47 | if err := f.Close(); err != nil { 48 | os.Remove(f.Name()) 49 | return false, err 50 | } 51 | return true, nil 52 | } 53 | 54 | func (s *Store) ReleaseByID(id string) (string, error) { 55 | fname := strings.TrimSpace(id) 56 | fpath := filepath.Join(s.dataDir, fname) 57 | 58 | data, err := ioutil.ReadFile(fpath) 59 | if err != nil { 60 | return "", err 61 | } 62 | 63 | if err := os.Remove(fpath); err != nil { 64 | return "", err 65 | } 66 | return strings.TrimSpace(string(data)), nil 67 | } 68 | -------------------------------------------------------------------------------- /ovs/backend/disk/backend_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | var store Store 9 | 10 | const ( 11 | ID = "testing-host" 12 | IFNAME = "veth" 13 | ) 14 | 15 | func TestNewStore(t *testing.T) { 16 | store, err := New("", "./test") 17 | assert.NotNil(t, store) 18 | assert.NoError(t, err) 19 | } 20 | 21 | func TestReserve(t *testing.T) { 22 | find, err := store.Reserve(ID, IFNAME) 23 | assert.True(t, find) 24 | assert.NoError(t, err) 25 | find, err = store.Reserve(ID, IFNAME) 26 | assert.False(t, find) 27 | assert.NoError(t, err) 28 | } 29 | 30 | func TestReleaseByID(t *testing.T) { 31 | name, err := store.ReleaseByID(ID) 32 | assert.NoError(t, err) 33 | assert.Equal(t, IFNAME, name) 34 | 35 | name, err = store.ReleaseByID(ID) 36 | assert.Error(t, err) 37 | } 38 | -------------------------------------------------------------------------------- /ovs/backend/store.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | type Store interface { 4 | Reserve(id, ovsIfaceName string) (bool, error) 5 | ReleaseByID(id string) (string, error) 6 | } 7 | -------------------------------------------------------------------------------- /ovs/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net" 8 | "runtime" 9 | "syscall" 10 | 11 | "github.com/John-Lin/ovs-cni/ovs/backend/disk" 12 | "github.com/containernetworking/cni/pkg/skel" 13 | "github.com/containernetworking/cni/pkg/types" 14 | "github.com/containernetworking/cni/pkg/types/current" 15 | "github.com/containernetworking/cni/pkg/version" 16 | "github.com/containernetworking/plugins/pkg/ip" 17 | "github.com/containernetworking/plugins/pkg/ipam" 18 | "github.com/containernetworking/plugins/pkg/ns" 19 | "github.com/containernetworking/plugins/pkg/utils" 20 | 21 | "github.com/j-keck/arping" 22 | log "github.com/sirupsen/logrus" 23 | "github.com/vishvananda/netlink" 24 | ) 25 | 26 | const defaultBrName = "br0" 27 | 28 | const defaultDataDir = "/var/lib/cni/networks" 29 | 30 | type NetConf struct { 31 | types.NetConf 32 | OVSBrName string `json:"ovsBridge"` 33 | IsGW bool `json:"isGateway"` 34 | IsDefaultGW bool `json:"isDefaultGateway"` 35 | IPMasq bool `json:"ipMasq"` 36 | VtepIPs []string `json:"vtepIPs"` 37 | Controller string `json:"controller,omitempty"` 38 | } 39 | 40 | type gwInfo struct { 41 | gws []net.IPNet 42 | family int 43 | defaultRouteFound bool 44 | } 45 | 46 | func init() { 47 | // this ensures that main runs only on main thread (thread group leader). 48 | // since namespace ops (unshare, setns) are done for a single thread, we 49 | // must ensure that the goroutine does not jump from OS thread to thread 50 | runtime.LockOSThread() 51 | } 52 | 53 | func loadNetConf(bytes []byte) (*NetConf, string, error) { 54 | n := &NetConf{ 55 | OVSBrName: defaultBrName, 56 | } 57 | if err := json.Unmarshal(bytes, n); err != nil { 58 | return nil, "", fmt.Errorf("failed to load netconf: %v", err) 59 | } 60 | return n, n.CNIVersion, nil 61 | } 62 | 63 | // calcGateways processes the results from the IPAM plugin and does the 64 | // following for each IP family: 65 | // - Calculates and compiles a list of gateway addresses 66 | // - Adds a default route if needed 67 | func calcGateways(result *current.Result, n *NetConf) (*gwInfo, *gwInfo, error) { 68 | 69 | gwsV4 := &gwInfo{} 70 | gwsV6 := &gwInfo{} 71 | 72 | for _, ipc := range result.IPs { 73 | 74 | // Determine if this config is IPv4 or IPv6 75 | var gws *gwInfo 76 | defaultNet := &net.IPNet{} 77 | switch { 78 | case ipc.Address.IP.To4() != nil: 79 | gws = gwsV4 80 | gws.family = netlink.FAMILY_V4 81 | defaultNet.IP = net.IPv4zero 82 | case len(ipc.Address.IP) == net.IPv6len: 83 | gws = gwsV6 84 | gws.family = netlink.FAMILY_V6 85 | defaultNet.IP = net.IPv6zero 86 | default: 87 | return nil, nil, fmt.Errorf("Unknown IP object: %v", ipc) 88 | } 89 | defaultNet.Mask = net.IPMask(defaultNet.IP) 90 | 91 | // All IPs currently refer to the container interface 92 | // Add the IP to the interface 93 | // 0 -> bridge itself 94 | // 1 -> veth endpoint 95 | // 2 -> interface in container 96 | ipc.Interface = current.Int(2) 97 | 98 | // If not provided, calculate the gateway address 99 | // We use first address of specific subnet as its gateway 100 | if ipc.Gateway == nil && n.IsGW { 101 | ipc.Gateway = getNextIP(&ipc.Address) 102 | } 103 | 104 | // Add a default route for this family using the current 105 | // gateway address if necessary. 106 | if n.IsDefaultGW && !gws.defaultRouteFound { 107 | for _, route := range result.Routes { 108 | if route.GW != nil && defaultNet.String() == route.Dst.String() { 109 | gws.defaultRouteFound = true 110 | break 111 | } 112 | } 113 | if !gws.defaultRouteFound { 114 | result.Routes = append( 115 | result.Routes, 116 | &types.Route{Dst: *defaultNet, GW: ipc.Gateway}, 117 | ) 118 | gws.defaultRouteFound = true 119 | } 120 | } 121 | 122 | // Append this gateway address to the list of gateways 123 | if n.IsGW { 124 | gw := net.IPNet{ 125 | IP: ipc.Gateway, 126 | Mask: ipc.Address.Mask, 127 | } 128 | gws.gws = append(gws.gws, gw) 129 | } 130 | } 131 | return gwsV4, gwsV6, nil 132 | } 133 | 134 | func ensureBridgeAddr(br *OVSSwitch, family int, ipn *net.IPNet) error { 135 | ovsbrLink, err := netlink.LinkByName(br.BridgeName) 136 | if err != nil { 137 | return fmt.Errorf("could not get ovs bridge link: %v", err) 138 | } 139 | 140 | addrs, err := netlink.AddrList(ovsbrLink, family) 141 | if err != nil && err != syscall.ENOENT { 142 | return fmt.Errorf("could not get list of IP addresses: %v", err) 143 | } 144 | 145 | ipnStr := ipn.String() 146 | for _, a := range addrs { 147 | // string comp is actually easiest for doing IPNet comps 148 | if a.IPNet.String() == ipnStr { 149 | return nil 150 | } 151 | } 152 | 153 | addr := &netlink.Addr{IPNet: ipn, Label: ""} 154 | if err := netlink.AddrAdd(ovsbrLink, addr); err != nil { 155 | return fmt.Errorf("could not add IP address to %q: %v", br.BridgeName, err) 156 | } 157 | 158 | // set ovs link device up 159 | if err := setLinkUp(br.BridgeName); err != nil { 160 | return fmt.Errorf("Error setting link %s up. Err: %v", br.BridgeName, err) 161 | } 162 | return nil 163 | } 164 | 165 | func setupVeth(netns ns.NetNS, br *OVSSwitch, ifName string, mtu int) (*current.Interface, *current.Interface, error) { 166 | contIface := ¤t.Interface{} 167 | hostIface := ¤t.Interface{} 168 | 169 | err := netns.Do(func(hostNS ns.NetNS) error { 170 | // create the veth pair in the container and move host end into host netns 171 | hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS) 172 | if err != nil { 173 | return err 174 | } 175 | contIface.Name = containerVeth.Name 176 | contIface.Mac = containerVeth.HardwareAddr.String() 177 | contIface.Sandbox = netns.Path() 178 | hostIface.Name = hostVeth.Name 179 | 180 | // ip link set lo up 181 | err = setLinkUp("lo") 182 | if err != nil { 183 | return err 184 | } 185 | return nil 186 | }) 187 | if err != nil { 188 | return nil, nil, err 189 | } 190 | 191 | err = br.addPort(hostIface.Name) 192 | if err != nil { 193 | log.Fatalf("failed to addPort switch - host: %v", err) 194 | } 195 | log.Infof("Adding a port for %s:", br.BridgeName) 196 | 197 | return hostIface, contIface, nil 198 | } 199 | 200 | func cmdAdd(args *skel.CmdArgs) error { 201 | n, cniVersion, err := loadNetConf(args.StdinData) 202 | if err != nil { 203 | return err 204 | } 205 | 206 | if n.IsDefaultGW { 207 | n.IsGW = true 208 | } 209 | 210 | // Create a Open vSwitch bridge 211 | br, brInterface, err := createOVS(n) 212 | if err != nil { 213 | return err 214 | } 215 | 216 | store, err := disk.New(n.OVSBrName, defaultDataDir) 217 | if err != nil { 218 | return err 219 | } 220 | 221 | netns, err := ns.GetNS(args.Netns) 222 | if err != nil { 223 | return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) 224 | } 225 | defer netns.Close() 226 | 227 | hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, 1400) 228 | if err != nil { 229 | return err 230 | } 231 | 232 | reserved, err := store.Reserve(args.ContainerID, hostInterface.Name) 233 | if err != nil { 234 | return err 235 | } 236 | if !reserved { 237 | return fmt.Errorf("requested interface name is not available") 238 | } 239 | 240 | // run the IPAM plugin and get back the config to apply 241 | r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) 242 | if err != nil { 243 | return err 244 | } 245 | 246 | // Convert whatever the IPAM result was into the current Result type 247 | result, err := current.NewResultFromResult(r) 248 | if err != nil { 249 | return err 250 | } 251 | 252 | if len(result.IPs) == 0 { 253 | return errors.New("IPAM plugin returned missing IP config") 254 | } 255 | 256 | result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface} 257 | 258 | // Gather gateway information for each IP family 259 | gwsV4, gwsV6, err := calcGateways(result, n) 260 | if err != nil { 261 | return err 262 | } 263 | 264 | // Configure the container hardware address and IP address(es) 265 | if err := netns.Do(func(_ ns.NetNS) error { 266 | contVeth, err := net.InterfaceByName(args.IfName) 267 | if err != nil { 268 | return err 269 | } 270 | 271 | if err := ipam.ConfigureIface(args.IfName, result); err != nil { 272 | return err 273 | } 274 | 275 | // Send a gratuitous arp 276 | for _, ipc := range result.IPs { 277 | if ipc.Version == "4" { 278 | _ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth) 279 | } 280 | } 281 | return nil 282 | }); err != nil { 283 | return err 284 | } 285 | 286 | if n.IsGW { 287 | var firstV4Addr net.IP 288 | // Set the IP address(es) on the bridge and enable forwarding 289 | for _, gws := range []*gwInfo{gwsV4, gwsV6} { 290 | for _, gw := range gws.gws { 291 | if gw.IP.To4() != nil && firstV4Addr == nil { 292 | firstV4Addr = gw.IP 293 | } 294 | 295 | err = ensureBridgeAddr(br, gws.family, &gw) 296 | if err != nil { 297 | return fmt.Errorf("failed to set bridge addr: %v", err) 298 | } 299 | } 300 | 301 | if gws.gws != nil { 302 | if err = enableIPForward(gws.family); err != nil { 303 | return fmt.Errorf("failed to enable forwarding: %v", err) 304 | } 305 | } 306 | } 307 | } 308 | 309 | if len(n.VtepIPs) != 0 { 310 | // Create VxLAN tunnelings 311 | if err = br.AddVTEPs(n.VtepIPs); err != nil { 312 | return err 313 | } 314 | } 315 | 316 | if n.Controller != "" { 317 | // Set a SDN controller 318 | if err = br.SetCtrl(n.Controller); err != nil { 319 | return err 320 | } 321 | } 322 | 323 | if n.IPMasq { 324 | chain := utils.FormatChainName(n.Name, args.ContainerID) 325 | comment := utils.FormatComment(n.Name, args.ContainerID) 326 | for _, ipc := range result.IPs { 327 | if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil { 328 | return err 329 | } 330 | } 331 | } 332 | 333 | return types.PrintResult(result, cniVersion) 334 | } 335 | 336 | func cmdDel(args *skel.CmdArgs) error { 337 | n, _, err := loadNetConf(args.StdinData) 338 | if err != nil { 339 | return err 340 | } 341 | 342 | if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { 343 | return err 344 | } 345 | 346 | if args.Netns == "" { 347 | return nil 348 | } 349 | store, err := disk.New(n.OVSBrName, defaultDataDir) 350 | if err != nil { 351 | return err 352 | } 353 | br, err := OVSByName(n.OVSBrName) 354 | if err != nil { 355 | return err 356 | } 357 | 358 | ovsInterface, err := store.ReleaseByID(args.ContainerID) 359 | if err != nil { 360 | return fmt.Errorf("released ovs interface is not available") 361 | } 362 | 363 | log.Infof("delete port from ovs interface name: %s", ovsInterface) 364 | err = br.delPort(ovsInterface) 365 | if err != nil { 366 | log.Fatalf("failed to delPort from switch %v", err) 367 | return err 368 | } 369 | // There is a netns so try to clean up. Delete can be called multiple times 370 | // so don't return an error if the device is already removed. 371 | // If the device isn't there then don't try to clean up IP masq either. 372 | var ipnets *net.IPNet 373 | err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { 374 | var err error 375 | ipnets, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_ALL) 376 | if err != nil && err == ip.ErrLinkNotFound { 377 | return nil 378 | } 379 | return err 380 | }) 381 | 382 | if err != nil { 383 | return err 384 | } 385 | 386 | if n.IPMasq { 387 | chain := utils.FormatChainName(n.Name, args.ContainerID) 388 | comment := utils.FormatComment(n.Name, args.ContainerID) 389 | if err := ip.TeardownIPMasq(ipnets, chain, comment); err != nil { 390 | return err 391 | } 392 | } 393 | 394 | return err 395 | } 396 | 397 | func main() { 398 | skel.PluginMain(cmdAdd, cmdDel, version.All) 399 | } 400 | -------------------------------------------------------------------------------- /ovs/ovs_switch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/John-Lin/ovsdb" 11 | "github.com/containernetworking/cni/pkg/types/current" 12 | ) 13 | 14 | // OVSSwitch is a bridge instance 15 | type OVSSwitch struct { 16 | NodeType string 17 | BridgeName string 18 | CtrlHostPort string 19 | ovsdb *ovsdb.OvsDriver 20 | } 21 | 22 | // NewOVSSwitch for creating a ovs bridge 23 | func NewOVSSwitch(bridgeName string) (*OVSSwitch, error) { 24 | sw := new(OVSSwitch) 25 | sw.NodeType = "OVSSwitch" 26 | sw.BridgeName = bridgeName 27 | 28 | sw.ovsdb = ovsdb.NewOvsDriverWithUnix(bridgeName) 29 | 30 | // Check if port is already part of the OVS and add it 31 | if !sw.ovsdb.IsPortNamePresent(bridgeName) { 32 | // Create an internal port in OVS 33 | err := sw.ovsdb.CreatePort(bridgeName, "internal", 0) 34 | if err != nil { 35 | return nil, err 36 | } 37 | } 38 | 39 | time.Sleep(300 * time.Millisecond) 40 | // log.Infof("Waiting for OVS bridge %s etup", bridgeName) 41 | 42 | // ip link set ovs up 43 | err := setLinkUp(bridgeName) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | return sw, nil 49 | } 50 | 51 | // addPort for asking OVSDB driver to add the port 52 | func (sw *OVSSwitch) addPort(ifName string) error { 53 | if !sw.ovsdb.IsPortNamePresent(ifName) { 54 | err := sw.ovsdb.CreatePort(ifName, "", 0) 55 | if err != nil { 56 | return fmt.Errorf("Error creating the port, Err: %v", err) 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | // delPort for asking OVSDB driver to delete the port 63 | func (sw *OVSSwitch) delPort(ifName string) error { 64 | if sw.ovsdb.IsPortNamePresent(ifName) { 65 | err := sw.ovsdb.DeletePort(ifName) 66 | if err != nil { 67 | return fmt.Errorf("Error deleting the port, Err: %v", err) 68 | } 69 | } 70 | return nil 71 | } 72 | 73 | // SetCtrl for seting up OpenFlow controller for ovs bridge 74 | func (sw *OVSSwitch) SetCtrl(hostport string) error { 75 | host, port, err := net.SplitHostPort(hostport) 76 | if err != nil { 77 | return fmt.Errorf("Invalid controller IP and port. Err: %v", err) 78 | } 79 | uPort, err := strconv.ParseUint(port, 10, 32) 80 | if err != nil { 81 | return fmt.Errorf("Invalid controller port number. Err: %v", err) 82 | } 83 | err = sw.ovsdb.AddController(host, uint16(uPort)) 84 | if err != nil { 85 | return fmt.Errorf("Error adding controller to OVS. Err: %v", err) 86 | } 87 | sw.CtrlHostPort = hostport 88 | return nil 89 | } 90 | 91 | func (sw *OVSSwitch) Delete() error { 92 | if exist := sw.ovsdb.IsBridgePresent(sw.BridgeName); exist != true { 93 | return errors.New(sw.BridgeName + " doesn't exist, we can delete") 94 | } 95 | 96 | return sw.ovsdb.DeleteBridge(sw.BridgeName) 97 | } 98 | 99 | func (sw *OVSSwitch) AddVTEPs(VtepIPs []string) error { 100 | for _, v := range VtepIPs { 101 | intfName := vxlanIfName(v) 102 | isPresent, vsifName := sw.ovsdb.IsVtepPresent(v) 103 | 104 | if !isPresent || (vsifName != intfName) { 105 | //create VTEP 106 | err := sw.ovsdb.CreateVtep(intfName, v) 107 | if err != nil { 108 | return fmt.Errorf("Error creating VTEP port %s. Err: %v", intfName, err) 109 | } 110 | 111 | } 112 | } 113 | return nil 114 | } 115 | 116 | // OVSByName is a alias for finding a ovs by name and returns a pointer to the object. 117 | func OVSByName(brName string) (*OVSSwitch, error) { 118 | return NewOVSSwitch(brName) 119 | } 120 | 121 | // createOVS is a helper function for create a ovs object 122 | func createOVS(n *NetConf) (*OVSSwitch, *current.Interface, error) { 123 | // create bridge if necessary 124 | ovsbr, err := NewOVSSwitch(n.OVSBrName) 125 | if err != nil { 126 | return nil, nil, fmt.Errorf("failed to setup bridge %q: %v", n.OVSBrName, err) 127 | } 128 | 129 | return ovsbr, ¤t.Interface{ 130 | Name: ovsbr.BridgeName, 131 | }, nil 132 | } 133 | -------------------------------------------------------------------------------- /ovs/ovs_switch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Che Wei, Lin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/stretchr/testify/assert" 19 | "testing" 20 | "time" 21 | ) 22 | 23 | var ovsSwitch *OVSSwitch 24 | var bridgeName string = "ovs_cni" 25 | 26 | func TestNewOVSSwitch(t *testing.T) { 27 | var err error 28 | ovsSwitch, err = NewOVSSwitch(bridgeName) 29 | assert.NoError(t, err) 30 | } 31 | 32 | func TestAddPort(t *testing.T) { 33 | err := ovsSwitch.addPort("test") 34 | assert.NoError(t, err) 35 | } 36 | 37 | func TestAddVTEPs(t *testing.T) { 38 | err := ovsSwitch.AddVTEPs([]string{"10.16.1.1"}) 39 | assert.NoError(t, err) 40 | } 41 | 42 | func TestAddPort_Invalid(t *testing.T) { 43 | err := ovsSwitch.addPort("") 44 | assert.Error(t, err) 45 | } 46 | 47 | func TestSetCtrl(t *testing.T) { 48 | err := ovsSwitch.SetCtrl("10.1.1.1:6653") 49 | assert.NoError(t, err) 50 | } 51 | 52 | func TestSetCtrl_Invalid(t *testing.T) { 53 | err := ovsSwitch.SetCtrl("abc") 54 | assert.Error(t, err) 55 | err = ovsSwitch.SetCtrl("10.1.1.1:abcde") 56 | assert.Error(t, err) 57 | } 58 | 59 | func TestDeleteOVSSwitch(t *testing.T) { 60 | err := ovsSwitch.Delete() 61 | assert.NoError(t, err) 62 | } 63 | 64 | func TestDeleteOVSSwitch_Invalid(t *testing.T) { 65 | //wait previous delete 66 | time.Sleep(1000 * time.Millisecond) 67 | err := ovsSwitch.Delete() 68 | assert.Error(t, err) 69 | } 70 | 71 | func TestCreateOVS(t *testing.T) { 72 | netConfig := NetConf{OVSBrName: "test0"} 73 | 74 | ovs, cT, err := createOVS(&netConfig) 75 | assert.NoError(t, err) 76 | assert.Equal(t, "test0", cT.Name) 77 | //wait previous delete 78 | time.Sleep(300 * time.Millisecond) 79 | ovs.Delete() 80 | } 81 | -------------------------------------------------------------------------------- /ovs/ovs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/stretchr/testify/assert" 19 | "testing" 20 | ) 21 | 22 | func TestLoadNetConf(t *testing.T) { 23 | 24 | t.Run("Valid", func(t *testing.T) { 25 | config := string(` 26 | { 27 | "name":"mynet", 28 | "cniVersion":"0.3.1", 29 | "type":"ovs", 30 | "ovsBridge":"br0", 31 | "isDefaultGateway": true, 32 | "ipMasq": true 33 | } 34 | `) 35 | 36 | n, s, err := loadNetConf([]byte(config)) 37 | assert.NoError(t, err) 38 | assert.Equal(t, s, "0.3.1") 39 | assert.Equal(t, "br0", n.OVSBrName) 40 | }) 41 | t.Run("InValid", func(t *testing.T) { 42 | config := string(` 43 | { 44 | asddad 45 | } 46 | `) 47 | 48 | n, s, err := loadNetConf([]byte(config)) 49 | assert.Error(t, err) 50 | assert.Nil(t, n) 51 | assert.Equal(t, s, "") 52 | }) 53 | 54 | } 55 | -------------------------------------------------------------------------------- /ovs/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Che Wei, Lin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | "net" 20 | "strings" 21 | 22 | "github.com/containernetworking/plugins/pkg/ip" 23 | "github.com/vishvananda/netlink" 24 | ) 25 | 26 | // vxlanIfName returns formatted vxlan interface name 27 | func vxlanIfName(vtepIP string) string { 28 | return fmt.Sprintf("vxif%s", strings.Replace(vtepIP, ".", "_", -1)) 29 | } 30 | 31 | // setLinkUp sets the link up 32 | func setLinkUp(name string) error { 33 | iface, err := netlink.LinkByName(name) 34 | if err != nil { 35 | return err 36 | } 37 | return netlink.LinkSetUp(iface) 38 | } 39 | 40 | func enableIPForward(family int) error { 41 | if family == netlink.FAMILY_V4 { 42 | return ip.EnableIP4Forward() 43 | } 44 | return ip.EnableIP6Forward() 45 | } 46 | 47 | //We use the first IP as gateway address 48 | func getNextIP(ipn *net.IPNet) net.IP { 49 | nid := ipn.IP.Mask(ipn.Mask) 50 | return ip.NextIP(nid) 51 | } 52 | -------------------------------------------------------------------------------- /ovs/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Che Wei, Lin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/stretchr/testify/assert" 19 | "github.com/vishvananda/netlink" 20 | "io/ioutil" 21 | "math/rand" 22 | "net" 23 | "os" 24 | "strconv" 25 | "strings" 26 | "testing" 27 | "time" 28 | ) 29 | 30 | func TestVxlanIfName(t *testing.T) { 31 | // Test to returns formatted vxlan interface name 32 | s1 := rand.NewSource(time.Now().UnixNano()) 33 | r1 := rand.New(s1) 34 | 35 | var reg []string 36 | for i := 0; i <= 3; i++ { 37 | reg = append(reg, strconv.Itoa(r1.Intn(256))) 38 | } 39 | 40 | intfName := vxlanIfName(strings.Join(reg[:], ".")) 41 | 42 | checked := "vxif" + strings.Join(reg[:], "_") 43 | assert.Equal(t, intfName, checked, "Those two names should be the same") 44 | } 45 | 46 | func TestSetLinkUp(t *testing.T) { 47 | err := setLinkUp("lo") 48 | assert.NoError(t, err) 49 | } 50 | 51 | func TestSetLinkUp_Invalid(t *testing.T) { 52 | err := setLinkUp("unknown") 53 | assert.Error(t, err) 54 | } 55 | 56 | func TestIPForward(t *testing.T) { 57 | v4Path := "/proc/sys/net/ipv4/ip_forward" 58 | v6Path := "/proc/sys/net/ipv6/conf/all/forwarding" 59 | 60 | if _, err := os.Stat(v4Path); !os.IsNotExist(err) { 61 | enableIPForward(netlink.FAMILY_V4) 62 | content, _ := ioutil.ReadFile(v4Path) 63 | assert.Equal(t, "1\n", string(content)) 64 | } 65 | 66 | if _, err := os.Stat(v6Path); !os.IsNotExist(err) { 67 | enableIPForward(netlink.FAMILY_V6) 68 | content, _ := ioutil.ReadFile(v6Path) 69 | assert.Equal(t, "1\n", string(content)) 70 | } 71 | } 72 | 73 | func TestGetGatewayFromIP(t *testing.T) { 74 | _, input, _ := net.ParseCIDR("192.168.194.0/22") 75 | 76 | gwIP := getNextIP(input) 77 | assert.Equal(t, gwIP.String(), "192.168.192.1") 78 | } 79 | -------------------------------------------------------------------------------- /vendor/vendor.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "", 3 | "ignore": "test", 4 | "package": [ 5 | { 6 | "checksumSHA1": "GI7WxdJmXR+08QSMzhf8dRj9ZC0=", 7 | "path": "github.com/John-Lin/ovsdb", 8 | "revision": "c84bb513c0524d0e891e45299560b178655c3241", 9 | "revisionTime": "2017-10-20T15:14:21Z" 10 | }, 11 | { 12 | "checksumSHA1": "72zRy5zCQvI87HDwbIfwhfOBXew=", 13 | "path": "github.com/cenkalti/hub", 14 | "revision": "11382a9960d39b0ecda16fd01c424c11ff765a34", 15 | "revisionTime": "2016-05-27T10:32:12Z" 16 | }, 17 | { 18 | "checksumSHA1": "xoOkcixbl+i6kkiHvXMNcN0/zJ0=", 19 | "path": "github.com/cenkalti/rpc2", 20 | "revision": "c51a77e5f664b4a8a69bce4270d14204f94d51f9", 21 | "revisionTime": "2017-07-26T07:05:24Z" 22 | }, 23 | { 24 | "checksumSHA1": "SV8XkMo9habP+09CWxEKm7ojrOA=", 25 | "path": "github.com/cenkalti/rpc2/jsonrpc", 26 | "revision": "c51a77e5f664b4a8a69bce4270d14204f94d51f9", 27 | "revisionTime": "2017-07-26T07:05:24Z" 28 | }, 29 | { 30 | "checksumSHA1": "Kh/msV2XTuYy0UB63hhoKcKfkyY=", 31 | "path": "github.com/containernetworking/cni/pkg/invoke", 32 | "revision": "ff7c3e02e3c212f63a642ad64a5ed22ee54450bd", 33 | "revisionTime": "2017-09-22T09:39:36Z" 34 | }, 35 | { 36 | "checksumSHA1": "LvjTFR2Am01kquJNDnfISncPfRw=", 37 | "path": "github.com/containernetworking/cni/pkg/skel", 38 | "revision": "a2da8f8d7fd8e6dc25f336408a8ac86f050fbd88", 39 | "revisionTime": "2017-07-05T15:32:19Z", 40 | "version": "=v0.6.0-rc1", 41 | "versionExact": "v0.6.0-rc1" 42 | }, 43 | { 44 | "checksumSHA1": "Sfdjk2zzDd0npAjZdULWjWoKVkQ=", 45 | "path": "github.com/containernetworking/cni/pkg/types", 46 | "revision": "a2da8f8d7fd8e6dc25f336408a8ac86f050fbd88", 47 | "revisionTime": "2017-07-05T15:32:19Z", 48 | "version": "=v0.6.0-rc1", 49 | "versionExact": "v0.6.0-rc1" 50 | }, 51 | { 52 | "checksumSHA1": "lmLYXYR/twmBu+6U678QNF+BQlQ=", 53 | "path": "github.com/containernetworking/cni/pkg/types/020", 54 | "revision": "ff7c3e02e3c212f63a642ad64a5ed22ee54450bd", 55 | "revisionTime": "2017-09-22T09:39:36Z" 56 | }, 57 | { 58 | "checksumSHA1": "r6qX7lyz4pweijiXjWMcxvm4wQQ=", 59 | "path": "github.com/containernetworking/cni/pkg/types/current", 60 | "revision": "a2da8f8d7fd8e6dc25f336408a8ac86f050fbd88", 61 | "revisionTime": "2017-07-05T15:32:19Z", 62 | "version": "=v0.6.0-rc1", 63 | "versionExact": "v0.6.0-rc1" 64 | }, 65 | { 66 | "checksumSHA1": "Y9KLxL9fFDnFHaDGOUfDkcbyV0U=", 67 | "path": "github.com/containernetworking/cni/pkg/version", 68 | "revision": "a2da8f8d7fd8e6dc25f336408a8ac86f050fbd88", 69 | "revisionTime": "2017-07-05T15:32:19Z", 70 | "version": "=v0.6.0-rc1", 71 | "versionExact": "v0.6.0-rc1" 72 | }, 73 | { 74 | "checksumSHA1": "BUj+UEIzHmUAfGD4c/srsQJ4TY8=", 75 | "path": "github.com/containernetworking/plugins/pkg/ip", 76 | "revision": "7480240de9749f9a0a5c8614b17f1f03e0c06ab9", 77 | "revisionTime": "2017-08-11T16:59:55Z", 78 | "version": "=v0.6.0", 79 | "versionExact": "v0.6.0" 80 | }, 81 | { 82 | "checksumSHA1": "kR8rb+saRyrxv5KqGcw4urd5Ugs=", 83 | "path": "github.com/containernetworking/plugins/pkg/ipam", 84 | "revision": "7480240de9749f9a0a5c8614b17f1f03e0c06ab9", 85 | "revisionTime": "2017-08-11T16:59:55Z", 86 | "version": "=v0.6.0", 87 | "versionExact": "v0.6.0" 88 | }, 89 | { 90 | "checksumSHA1": "wmZ5nRpAN/myLfAzVpPR1RxnoFQ=", 91 | "path": "github.com/containernetworking/plugins/pkg/ns", 92 | "revision": "7480240de9749f9a0a5c8614b17f1f03e0c06ab9", 93 | "revisionTime": "2017-08-11T16:59:55Z", 94 | "version": "=v0.6.0", 95 | "versionExact": "v0.6.0" 96 | }, 97 | { 98 | "checksumSHA1": "909B5R5pbQLe7y6cgMlr8d0aLO0=", 99 | "path": "github.com/containernetworking/plugins/pkg/utils", 100 | "revision": "7480240de9749f9a0a5c8614b17f1f03e0c06ab9", 101 | "revisionTime": "2017-08-11T16:59:55Z", 102 | "version": "=v0.6.0", 103 | "versionExact": "v0.6.0" 104 | }, 105 | { 106 | "checksumSHA1": "rUAk+nce6eDIWtYo/QYIoqlTfrY=", 107 | "path": "github.com/containernetworking/plugins/pkg/utils/hwaddr", 108 | "revision": "0063a1b9d0b137ac5b1b4bdbef9ba75e2e76ebc7", 109 | "revisionTime": "2017-10-11T13:50:19Z" 110 | }, 111 | { 112 | "checksumSHA1": "7BC2/27NId9xaPDB5w3nWN2mn9A=", 113 | "path": "github.com/coreos/etcd/auth/authpb", 114 | "revision": "f8bec0f631a98da73fcc57367542d6009bd2cbee", 115 | "revisionTime": "2017-11-02T15:35:43Z" 116 | }, 117 | { 118 | "checksumSHA1": "4m3qRCZfmTyRSzd4eH8ovMb5OXY=", 119 | "path": "github.com/coreos/etcd/clientv3", 120 | "revision": "f8bec0f631a98da73fcc57367542d6009bd2cbee", 121 | "revisionTime": "2017-11-02T15:35:43Z" 122 | }, 123 | { 124 | "checksumSHA1": "VMC9J0rMVk3Fv8r8Bj7qqLlXc3E=", 125 | "path": "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes", 126 | "revision": "f8bec0f631a98da73fcc57367542d6009bd2cbee", 127 | "revisionTime": "2017-11-02T15:35:43Z" 128 | }, 129 | { 130 | "checksumSHA1": "c0ltvGUOnk8qaEshFwc0PDH5nbc=", 131 | "path": "github.com/coreos/etcd/etcdserver/etcdserverpb", 132 | "revision": "f8bec0f631a98da73fcc57367542d6009bd2cbee", 133 | "revisionTime": "2017-11-02T15:35:43Z" 134 | }, 135 | { 136 | "checksumSHA1": "JAkX9DfIBrSe0vUa07xl5cikxVQ=", 137 | "path": "github.com/coreos/etcd/mvcc/mvccpb", 138 | "revision": "f8bec0f631a98da73fcc57367542d6009bd2cbee", 139 | "revisionTime": "2017-11-02T15:35:43Z" 140 | }, 141 | { 142 | "checksumSHA1": "rMyIh9PsSvPs6Yd+YgKITQzQJx8=", 143 | "path": "github.com/coreos/etcd/pkg/tlsutil", 144 | "revision": "6a265731e10a5137b991c1aa3a83ecefdd149d50", 145 | "revisionTime": "2018-02-02T22:08:29Z" 146 | }, 147 | { 148 | "checksumSHA1": "PoTXn5yRFx7RyFeBgjoPoAbx2C0=", 149 | "path": "github.com/coreos/etcd/pkg/transport", 150 | "revision": "6a265731e10a5137b991c1aa3a83ecefdd149d50", 151 | "revisionTime": "2018-02-02T22:08:29Z" 152 | }, 153 | { 154 | "checksumSHA1": "RM7wQmNoUSB93ogKxdpA4265++Y=", 155 | "path": "github.com/coreos/go-iptables/iptables", 156 | "revision": "17b936e6ccb6f6e424f7d89c614164e796df1661", 157 | "revisionTime": "2017-09-15T11:44:21Z" 158 | }, 159 | { 160 | "checksumSHA1": "mrz/kicZiUaHxkyfvC/DyQcr8Do=", 161 | "path": "github.com/davecgh/go-spew/spew", 162 | "revision": "ecdeabc65495df2dec95d7c4a4c3e021903035e5", 163 | "revisionTime": "2017-10-02T20:02:53Z" 164 | }, 165 | { 166 | "checksumSHA1": "yqF125xVSkmfLpIVGrLlfE05IUk=", 167 | "path": "github.com/golang/protobuf/proto", 168 | "revision": "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9", 169 | "revisionTime": "2017-10-21T04:39:52Z" 170 | }, 171 | { 172 | "checksumSHA1": "VfkiItDBFFkZluaAMAzJipDXNBY=", 173 | "path": "github.com/golang/protobuf/ptypes", 174 | "revision": "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9", 175 | "revisionTime": "2017-10-21T04:39:52Z" 176 | }, 177 | { 178 | "checksumSHA1": "UB9scpDxeFjQe5tEthuR4zCLRu4=", 179 | "path": "github.com/golang/protobuf/ptypes/any", 180 | "revision": "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9", 181 | "revisionTime": "2017-10-21T04:39:52Z" 182 | }, 183 | { 184 | "checksumSHA1": "hUjAj0dheFVDl84BAnSWj9qy2iY=", 185 | "path": "github.com/golang/protobuf/ptypes/duration", 186 | "revision": "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9", 187 | "revisionTime": "2017-10-21T04:39:52Z" 188 | }, 189 | { 190 | "checksumSHA1": "O2ItP5rmfrgxPufhjJXbFlXuyL8=", 191 | "path": "github.com/golang/protobuf/ptypes/timestamp", 192 | "revision": "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9", 193 | "revisionTime": "2017-10-21T04:39:52Z" 194 | }, 195 | { 196 | "checksumSHA1": "p6Lp0JStM2aO1Jns3ZHixwpVXLY=", 197 | "path": "github.com/j-keck/arping", 198 | "revision": "2cf9dc699c5640a7e2c81403a44127bf28033600", 199 | "revisionTime": "2016-06-18T11:04:41Z" 200 | }, 201 | { 202 | "checksumSHA1": "LuFv4/jlrmFNnDb/5SCSEPAM9vU=", 203 | "path": "github.com/pmezard/go-difflib/difflib", 204 | "revision": "792786c7400a136282c1664665ae0a8db921c6c2", 205 | "revisionTime": "2016-01-10T10:55:54Z" 206 | }, 207 | { 208 | "checksumSHA1": "BYvROBsiyAXK4sq6yhDe8RgT4LM=", 209 | "path": "github.com/sirupsen/logrus", 210 | "revision": "89742aefa4b206dcf400792f3bd35b542998eb3b", 211 | "revisionTime": "2017-08-22T13:27:46Z" 212 | }, 213 | { 214 | "checksumSHA1": "i1IXs5Zp2VX1F8Y1KXFHcoC8R9c=", 215 | "path": "github.com/socketplane/libovsdb", 216 | "revision": "4de3618546deba09d8875d719752db32bd4652c0", 217 | "revisionTime": "2017-01-16T17:48:20Z" 218 | }, 219 | { 220 | "checksumSHA1": "mGbTYZ8dHVTiPTTJu3ktp+84pPI=", 221 | "path": "github.com/stretchr/testify/assert", 222 | "revision": "890a5c3458b43e6104ff5da8dfa139d013d77544", 223 | "revisionTime": "2017-07-05T02:17:15Z" 224 | }, 225 | { 226 | "checksumSHA1": "t1ctd42IxJTpv7KWicxdmEeb+KE=", 227 | "path": "github.com/vishvananda/netlink", 228 | "revision": "177f1ceba557262b3f1c3aba4df93a29199fb4eb", 229 | "revisionTime": "2017-09-15T20:12:01Z" 230 | }, 231 | { 232 | "checksumSHA1": "oNtzRasYc/H9quYOt/bNVP814OU=", 233 | "path": "github.com/vishvananda/netlink/nl", 234 | "revision": "177f1ceba557262b3f1c3aba4df93a29199fb4eb", 235 | "revisionTime": "2017-09-15T20:12:01Z" 236 | }, 237 | { 238 | "checksumSHA1": "YDN9yoXZ/KBWqILw9PCNtd/SRAI=", 239 | "path": "github.com/vishvananda/netns", 240 | "revision": "86bef332bfc3b59b7624a600bd53009ce91a9829", 241 | "revisionTime": "2017-07-07T00:42:18Z" 242 | }, 243 | { 244 | "checksumSHA1": "nqWNlnMmVpt628zzvyo6Yv2CX5Q=", 245 | "path": "golang.org/x/crypto/ssh/terminal", 246 | "revision": "9419663f5a44be8b34ca85f08abc5fe1be11f8a3", 247 | "revisionTime": "2017-09-30T17:45:11Z" 248 | }, 249 | { 250 | "checksumSHA1": "GtamqiJoL7PGHsN454AoffBFMa8=", 251 | "path": "golang.org/x/net/context", 252 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 253 | "revisionTime": "2017-11-01T17:47:04Z" 254 | }, 255 | { 256 | "checksumSHA1": "zZG+ad2/41l7WmUsh2qkgUT2mLM=", 257 | "path": "golang.org/x/net/http2", 258 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 259 | "revisionTime": "2017-11-01T17:47:04Z" 260 | }, 261 | { 262 | "checksumSHA1": "ezWhc7n/FtqkLDQKeU2JbW+80tE=", 263 | "path": "golang.org/x/net/http2/hpack", 264 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 265 | "revisionTime": "2017-11-01T17:47:04Z" 266 | }, 267 | { 268 | "checksumSHA1": "RcrB7tgYS/GMW4QrwVdMOTNqIU8=", 269 | "path": "golang.org/x/net/idna", 270 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 271 | "revisionTime": "2017-11-01T17:47:04Z" 272 | }, 273 | { 274 | "checksumSHA1": "UxahDzW2v4mf/+aFxruuupaoIwo=", 275 | "path": "golang.org/x/net/internal/timeseries", 276 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 277 | "revisionTime": "2017-11-01T17:47:04Z" 278 | }, 279 | { 280 | "checksumSHA1": "3xyuaSNmClqG4YWC7g0isQIbUTc=", 281 | "path": "golang.org/x/net/lex/httplex", 282 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 283 | "revisionTime": "2017-11-01T17:47:04Z" 284 | }, 285 | { 286 | "checksumSHA1": "u/r66lwYfgg682u5hZG7/E7+VCY=", 287 | "path": "golang.org/x/net/trace", 288 | "revision": "49e6db1c9ed2b2fdbc57fd579f0ca8f6082350be", 289 | "revisionTime": "2017-11-01T17:47:04Z" 290 | }, 291 | { 292 | "checksumSHA1": "fdCrXJR1ntbZ1a2lrAMdZk2fWR4=", 293 | "path": "golang.org/x/sys/unix", 294 | "revision": "43eea11bc92608addb41b8a406b0407495c106f6", 295 | "revisionTime": "2017-10-12T11:34:53Z" 296 | }, 297 | { 298 | "checksumSHA1": "pBPFzDGt3AVSRffB7ffiUnruFUk=", 299 | "path": "golang.org/x/sys/windows", 300 | "revision": "43eea11bc92608addb41b8a406b0407495c106f6", 301 | "revisionTime": "2017-10-12T11:34:53Z" 302 | }, 303 | { 304 | "checksumSHA1": "tltivJ/uj/lqLk05IqGfCv2F/E8=", 305 | "path": "golang.org/x/text/secure/bidirule", 306 | "revision": "2df67286dd25dd6a99db07737c3e6620378a97a3", 307 | "revisionTime": "2017-11-02T14:38:37Z" 308 | }, 309 | { 310 | "checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=", 311 | "path": "golang.org/x/text/transform", 312 | "revision": "2df67286dd25dd6a99db07737c3e6620378a97a3", 313 | "revisionTime": "2017-11-02T14:38:37Z" 314 | }, 315 | { 316 | "checksumSHA1": "iB6/RoQIzBaZxVi+t7tzbkwZTlo=", 317 | "path": "golang.org/x/text/unicode/bidi", 318 | "revision": "2df67286dd25dd6a99db07737c3e6620378a97a3", 319 | "revisionTime": "2017-11-02T14:38:37Z" 320 | }, 321 | { 322 | "checksumSHA1": "km/8bLtOpIP7sua4MnEmiSDYTAE=", 323 | "path": "golang.org/x/text/unicode/norm", 324 | "revision": "2df67286dd25dd6a99db07737c3e6620378a97a3", 325 | "revisionTime": "2017-11-02T14:38:37Z" 326 | }, 327 | { 328 | "checksumSHA1": "Tc3BU26zThLzcyqbVtiSEp7EpU8=", 329 | "path": "google.golang.org/genproto/googleapis/rpc/status", 330 | "revision": "03951f3b52582ecab79e5a37a86b5a20c5f64df7", 331 | "revisionTime": "2017-10-31T22:45:10Z" 332 | }, 333 | { 334 | "checksumSHA1": "DPO9OQy3rQEtZHHsJQw7TyH8dU4=", 335 | "path": "google.golang.org/grpc", 336 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 337 | "revisionTime": "2017-11-02T16:56:04Z" 338 | }, 339 | { 340 | "checksumSHA1": "HoJvHF9RxOinJPAAbAhfZSNUxBY=", 341 | "path": "google.golang.org/grpc/balancer", 342 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 343 | "revisionTime": "2017-11-02T16:56:04Z" 344 | }, 345 | { 346 | "checksumSHA1": "os98urLvZVriKRbHhIsipJfcT7Q=", 347 | "path": "google.golang.org/grpc/balancer/roundrobin", 348 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 349 | "revisionTime": "2017-11-02T16:56:04Z" 350 | }, 351 | { 352 | "checksumSHA1": "Dkjgw1HasWvqct0IuiZdjbD7O0c=", 353 | "path": "google.golang.org/grpc/codes", 354 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 355 | "revisionTime": "2017-11-02T16:56:04Z" 356 | }, 357 | { 358 | "checksumSHA1": "XH2WYcDNwVO47zYShREJjcYXm0Y=", 359 | "path": "google.golang.org/grpc/connectivity", 360 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 361 | "revisionTime": "2017-11-02T16:56:04Z" 362 | }, 363 | { 364 | "checksumSHA1": "EEpFv96tNVcG9Z42ehroNbG/VKI=", 365 | "path": "google.golang.org/grpc/credentials", 366 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 367 | "revisionTime": "2017-11-02T16:56:04Z" 368 | }, 369 | { 370 | "checksumSHA1": "k3l7Hrce7IiDOzDlF4UDJ4fs2Bc=", 371 | "path": "google.golang.org/grpc/encoding", 372 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 373 | "revisionTime": "2017-11-02T16:56:04Z" 374 | }, 375 | { 376 | "checksumSHA1": "H7SuPUqbPcdbNqgl+k3ohuwMAwE=", 377 | "path": "google.golang.org/grpc/grpclb/grpc_lb_v1/messages", 378 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 379 | "revisionTime": "2017-11-02T16:56:04Z" 380 | }, 381 | { 382 | "checksumSHA1": "ntHev01vgZgeIh5VFRmbLx/BSTo=", 383 | "path": "google.golang.org/grpc/grpclog", 384 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 385 | "revisionTime": "2017-11-02T16:56:04Z" 386 | }, 387 | { 388 | "checksumSHA1": "6vY7tYjV84pnr3sDctzx53Bs8b0=", 389 | "path": "google.golang.org/grpc/health/grpc_health_v1", 390 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 391 | "revisionTime": "2017-11-02T16:56:04Z" 392 | }, 393 | { 394 | "checksumSHA1": "U9vDe05/tQrvFBojOQX8Xk12W9I=", 395 | "path": "google.golang.org/grpc/internal", 396 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 397 | "revisionTime": "2017-11-02T16:56:04Z" 398 | }, 399 | { 400 | "checksumSHA1": "hcuHgKp8W0wIzoCnNfKI8NUss5o=", 401 | "path": "google.golang.org/grpc/keepalive", 402 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 403 | "revisionTime": "2017-11-02T16:56:04Z" 404 | }, 405 | { 406 | "checksumSHA1": "KeUmTZV+2X46C49cKyjp+xM7fvw=", 407 | "path": "google.golang.org/grpc/metadata", 408 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 409 | "revisionTime": "2017-11-02T16:56:04Z" 410 | }, 411 | { 412 | "checksumSHA1": "dgwdT20kXe4ZbXBOFbTwVQt8rmA=", 413 | "path": "google.golang.org/grpc/naming", 414 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 415 | "revisionTime": "2017-11-02T16:56:04Z" 416 | }, 417 | { 418 | "checksumSHA1": "n5EgDdBqFMa2KQFhtl+FF/4gIFo=", 419 | "path": "google.golang.org/grpc/peer", 420 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 421 | "revisionTime": "2017-11-02T16:56:04Z" 422 | }, 423 | { 424 | "checksumSHA1": "H7VyP18nJ9MmoB5r9+I7EKVEeVM=", 425 | "path": "google.golang.org/grpc/resolver", 426 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 427 | "revisionTime": "2017-11-02T16:56:04Z" 428 | }, 429 | { 430 | "checksumSHA1": "WpWF+bDzObsHf+bjoGpb/abeFxo=", 431 | "path": "google.golang.org/grpc/resolver/dns", 432 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 433 | "revisionTime": "2017-11-02T16:56:04Z" 434 | }, 435 | { 436 | "checksumSHA1": "zs9M4xE8Lyg4wvuYvR00XoBxmuw=", 437 | "path": "google.golang.org/grpc/resolver/passthrough", 438 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 439 | "revisionTime": "2017-11-02T16:56:04Z" 440 | }, 441 | { 442 | "checksumSHA1": "G9lgXNi7qClo5sM2s6TbTHLFR3g=", 443 | "path": "google.golang.org/grpc/stats", 444 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 445 | "revisionTime": "2017-11-02T16:56:04Z" 446 | }, 447 | { 448 | "checksumSHA1": "3Dwz4RLstDHMPyDA7BUsYe+JP4w=", 449 | "path": "google.golang.org/grpc/status", 450 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 451 | "revisionTime": "2017-11-02T16:56:04Z" 452 | }, 453 | { 454 | "checksumSHA1": "qvArRhlrww5WvRmbyMF2mUfbJew=", 455 | "path": "google.golang.org/grpc/tap", 456 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 457 | "revisionTime": "2017-11-02T16:56:04Z" 458 | }, 459 | { 460 | "checksumSHA1": "3uvwXkHtrY5l9RU1Udc5nna863o=", 461 | "path": "google.golang.org/grpc/transport", 462 | "revision": "af224a8a485281b0d4e1e285642fcbaa8fe8fb1e", 463 | "revisionTime": "2017-11-02T16:56:04Z" 464 | } 465 | ], 466 | "rootPath": "github.com/John-Lin/ovs-cni" 467 | } 468 | --------------------------------------------------------------------------------