├── .dockerignore ├── .github └── workflows │ └── actions.yml ├── .gitignore ├── .golangci.yml ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── aws ├── allocate.go ├── bugs.go ├── cache │ ├── cacheable.go │ └── cacheable_test.go ├── client.go ├── client_test.go ├── freeip.go ├── interface.go ├── interface_test.go ├── limits.go ├── limits_test.go ├── metadata.go ├── registry.go ├── registry_test.go ├── subnets.go ├── subnets_test.go ├── util.go ├── util_test.go └── vpc.go ├── cmd └── cni-ipvlan-vpc-k8s-tool │ ├── cni-ipvlan-vpc-k8s-tool.go │ └── cni-ipvlan-vpc-k8s-tool_test.go ├── doc.go ├── docs ├── cni.svg └── internet-egress.svg ├── go.mod ├── go.sum ├── lib ├── jsontime.go ├── jsontime_test.go └── lock.go ├── nl ├── desc.go ├── doc.go ├── docker.go ├── down.go ├── down_test.go ├── mtu.go ├── up.go └── up_test.go └── plugin ├── ipam └── main.go ├── ipvlan └── ipvlan.go └── unnumbered-ptp └── unnumbered-ptp.go /.dockerignore: -------------------------------------------------------------------------------- 1 | cni-eni 2 | *~ 3 | .vscode 4 | .idea 5 | vendor/ 6 | cni-eni-tool 7 | eni-ipam 8 | eni-ipvlan 9 | *.tar.gz 10 | -------------------------------------------------------------------------------- /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | name: go 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - uses: actions/setup-go@v2 16 | with: 17 | go-version: 1.17.1 18 | - name: go mod cache 19 | uses: actions/cache@v2 20 | with: 21 | path: | 22 | ~/go/pkg/mod 23 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 24 | restore-keys: | 25 | ${{ runner.os }}-go-mod- 26 | - name: go build cache 27 | uses: actions/cache@v2 28 | with: 29 | path: | 30 | ~/.cache/go-build 31 | key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} 32 | restore-keys: | 33 | ${{ runner.os }}-go-build- 34 | - run: make build 35 | 36 | test: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v2 40 | - uses: actions/setup-go@v2 41 | with: 42 | go-version: 1.17.1 43 | - name: go mod cache 44 | uses: actions/cache@v2 45 | with: 46 | path: | 47 | ~/go/pkg/mod 48 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 49 | restore-keys: | 50 | ${{ runner.os }}-go-mod- 51 | - name: go test cache 52 | uses: actions/cache@v2 53 | with: 54 | path: | 55 | ~/.cache/go-build 56 | key: ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }} 57 | restore-keys: | 58 | ${{ runner.os }}-go-test- 59 | - run: sudo make test 60 | 61 | lint: 62 | runs-on: ubuntu-latest 63 | steps: 64 | - uses: actions/checkout@v2 65 | - uses: actions/setup-go@v2 66 | with: 67 | go-version: 1.17.1 68 | - name: go mod cache 69 | uses: actions/cache@v2 70 | with: 71 | path: | 72 | ~/go/pkg/mod 73 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 74 | restore-keys: | 75 | ${{ runner.os }}-go-mod- 76 | - name: go lint cache 77 | uses: actions/cache@v2 78 | with: 79 | path: | 80 | ~/.cache/go-build 81 | ~/.cache/golangci-lint 82 | key: ${{ runner.os }}-go-lint-${{ hashFiles('**/go.sum') }} 83 | restore-keys: | 84 | ${{ runner.os }}-go-lint- 85 | - name: golangci-lint 86 | uses: golangci/golangci-lint-action@v2 87 | with: 88 | version: v1.30 89 | - run: make lint 90 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .idea 3 | .vscode 4 | vendor/ 5 | vendor.orig/ 6 | 7 | /cni-ipvlan-vpc-k8s-* 8 | *.tar.gz 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | disable-all: true 3 | enable: 4 | - deadcode 5 | - errcheck 6 | - goconst 7 | - goimports 8 | - golint 9 | - gosec 10 | - gosimple 11 | - govet 12 | - ineffassign 13 | - misspell 14 | - nakedret 15 | - staticcheck 16 | - structcheck 17 | - typecheck 18 | - unconvert 19 | - varcheck 20 | linters-settings: 21 | misspell: 22 | locale: US 23 | 24 | run: 25 | deadline: 1m 26 | 27 | issues: 28 | exclude: 29 | - G404 # crypto rand warning, we don't need secure random numbers 30 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | This project is governed by [Lyft's code of conduct](https://github.com/lyft/code-of-conduct). 2 | All contributors and participants agree to abide by its terms. 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10.3 AS builder 2 | LABEL maintainer="mcutalo@lyft.com" 3 | 4 | WORKDIR /go/src/github.com/lyft/cni-ipvlan-vpc-k8s/ 5 | 6 | RUN go get github.com/golang/dep && \ 7 | go install github.com/golang/dep/cmd/dep && \ 8 | go get -u gopkg.in/alecthomas/gometalinter.v2 && \ 9 | gometalinter.v2 --install 10 | 11 | COPY . /go/src/github.com/lyft/cni-ipvlan-vpc-k8s/ 12 | 13 | RUN dep ensure -v && make build 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "{}" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2017 Lyft, Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CGO_ENABLED=1 2 | export CGO_ENABLED 3 | NAME=cni-ipvlan-vpc-k8s 4 | VERSION:=$(shell git describe --tags) 5 | DOCKER_IMAGE=lyft/cni-ipvlan-vpc-k8s:$(VERSION) 6 | export GO111MODULE=on 7 | 8 | .PHONY: all 9 | all: build test 10 | 11 | .PHONY: clean 12 | clean: 13 | rm -f *.tar.gz $(NAME)-* 14 | 15 | .PHONY: lint 16 | lint: 17 | golangci-lint run ./... 18 | 19 | .PHONY: test 20 | test: 21 | ifndef GOOS 22 | go test -v ./aws/... ./nl ./cmd/cni-ipvlan-vpc-k8s-tool ./lib/... 23 | else 24 | @echo Tests not available when cross-compiling 25 | endif 26 | 27 | .PHONY: build 28 | build: 29 | go build -i -o $(NAME)-ipam ./plugin/ipam/main.go 30 | go build -i -o $(NAME)-ipvlan ./plugin/ipvlan/ipvlan.go 31 | go build -i -o $(NAME)-unnumbered-ptp ./plugin/unnumbered-ptp/unnumbered-ptp.go 32 | go build -i -ldflags "-X main.version=$(VERSION)" -o $(NAME)-tool ./cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go 33 | 34 | tar cvzf cni-ipvlan-vpc-k8s-${GOARCH}-$(VERSION).tar.gz $(NAME)-ipam $(NAME)-ipvlan $(NAME)-unnumbered-ptp $(NAME)-tool 35 | 36 | .PHONY: test-docker 37 | test-docker: 38 | docker build -t $(DOCKER_IMAGE) . 39 | 40 | .PHONY: build-docker 41 | build-docker: test-docker 42 | docker run --rm -v $(PWD):/dist:rw $(DOCKER_IMAGE) bash -exc 'cp /go/src/github.com/lyft/cni-ipvlan-vpc-k8s/cni-ipvlan-vpc-k8s-$(VERSION).tar.gz /dist' 43 | 44 | .PHONY: interactive-docker 45 | interactive-docker: test-docker 46 | docker run --privileged -v $(PWD):/go/src/github.com/lyft/cni-ipvlan-vpc-k8s -it $(DOCKER_IMAGE) /bin/bash 47 | 48 | .PHONY: ci 49 | ci: 50 | go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.30.0 51 | $(MAKE) all 52 | $(MAKE) lint 53 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | cni-ipvlan-vpc-k8s 2 | Copyright 2017 Lyft Inc. 3 | 4 | This product includes software developed at Lyft Inc. 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cni-ipvlan-vpc-k8s: IPvlan Overlay-free Kubernetes Networking in AWS 2 | 3 | `cni-ipvlan-vpc-k8s` contains a set of 4 | [CNI](https://github.com/containernetworking/cni) and IPAM plugins to 5 | provide a simple, host-local, low latency, high throughput, and [compliant 6 | networking stack for 7 | Kubernetes](https://kubernetes.io/docs/concepts/cluster-administration/networking/#kubernetes-model) 8 | within [Amazon Virtual Private Cloud 9 | (VPC)](https://aws.amazon.com/vpc/) environments by making use of 10 | [Amazon Elastic Network Interfaces 11 | (ENI)](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html) 12 | and binding AWS-managed IPs into Pods using the Linux kernel's IPvlan 13 | driver in L2 mode. 14 | 15 | The plugins are designed to be straightforward to configure and deploy 16 | within a VPC. Kubelets boot and then self-configure and scale their IP 17 | usage as needed, without requiring the often recommended complexities 18 | of administering overlay networks, BGP, disabling source/destination 19 | checks, or adjusting VPC route tables to provide per-instance subnets 20 | to each host (which is limited to 50-100 entries per VPC). In short, 21 | `cni-ipvlan-vpc-k8s` significantly reduces the network complexity 22 | required to deploy Kubernetes at scale within AWS. 23 | 24 | The maximum number of Pods per AWS instance is determined by [ENI 25 | limits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI). Instance 26 | types offering 8 ENIs can scale up to and beyond the default 27 | Kubernetes limit of 110 pods per instance. 28 | 29 | ## Features 30 | 31 | * Designed and tested on Kubernetes in AWS (v1.10 with cri-o, docker, and containerd) 32 | * No overlay network; very low overhead with IPvlan 33 | * No external or local network services required outside of the AWS 34 | EC2 API; host-local scale up and scale down of network resources 35 | * Unnumbered point-to-point interfaces connect Pods with their Kubelet 36 | and Daemon Sets using their well-known Kubernetes IPs and optionally 37 | provide IPv4 internet connectivity via NAT by directing traffic over 38 | the primary private IP of the boot ENI making use of Amazon's Public 39 | IPv4 addressing attribute feature. 40 | * No asymmetric routing; no VPC routing table changes required 41 | * Pod IPs are directly addressable from non-Kubernetes VPC 42 | hosts, easing migration of existing pre-Kubernetes service meshes 43 | and infrastructure. 44 | * Automatic discovery of AWS resources, minimal plugin configuration 45 | required. 46 | 47 | ## How it Works 48 | 49 | The primary EC2 boot ENI with its primary private IP is used as the IP 50 | address for the node. Our CNI plugins manage additional ENIs and 51 | private IPs on those ENIs to assign IP addresses to Pods. 52 | 53 | Each Pod contains two network interfaces, a primary IPvlan interface 54 | and an unnumbered point-to-point virtual ethernet interface. These 55 | interfaces are created via a chained CNI execution. 56 | 57 | ![CNI Overview Diagram](./docs/cni.svg) 58 | 59 | * IPvlan interface: The IPvlan interface with the Pod’s IP is used for 60 | all VPC traffic and provides minimal overhead for network packet 61 | processing within the Linux kernel. The master device is the ENI of 62 | the associated Pod IP. IPvlan is used in L2 mode with isolation 63 | provided from all other ENIs, including the boot ENI handling 64 | traffic for the Kubernetes control plane. 65 | * Unnumbered point-to-point interface: A pair of virtual ethernet 66 | interfaces (veth) without IP addresses is used to interconnect the 67 | Pod’s network namespace to the default network namespace. The 68 | interface is used as the default route (non-VPC traffic) from the 69 | Pod and additional routes are created on each side to direct traffic 70 | between the node IP and the Pod IP over the link. For traffic sent 71 | over the interface, the Linux kernel borrows the IP address from the 72 | IPvlan interface for the Pod side and the boot ENI interface for the 73 | Kubelet side. Kubernetes Pods and nodes communicate using the same 74 | well-known addresses regardless of which interface (IPvlan or veth) 75 | is used for communication. This particular trick of “IP unnumbered 76 | configuration” is documented in 77 | [RFC5309](https://tools.ietf.org/html/rfc5309). 78 | 79 | ### Internet egress 80 | For applications where Pods need to directly communicate with the 81 | Internet, by setting the default route to the unnumbered 82 | point-to-point interface, our stack can source NAT traffic from the 83 | Pod over the primary private IP of the boot ENI, which enables making 84 | use of Amazon’s Public IPv4 addressing attribute feature. When 85 | enabled, Pods can egress to the Internet without needing to manage 86 | Elastic IPs or NAT Gateways. 87 | 88 | ![CNI Overview Diagram](./docs/internet-egress.svg) 89 | 90 | 91 | ### Host namespace interconnect 92 | Kubelets and Daemon Sets have high bandwidth, host-local access to all 93 | Pods running on the instance — traffic doesn’t transit ENI 94 | devices. Source and destination IPs are the well-known Kubernetes 95 | addresses on either side of the connect. 96 | 97 | * kube-proxy: We use kube-proxy in iptables mode and it functions as 98 | expected. The Pod's source IP is retained -- Kubernetes Services see 99 | connections from the Pod's source IP. The unnumbered point-to-point 100 | interface is used to loop traffic between kube-proxy in the default 101 | namespace for outbound connections created in the Pod namespace. 102 | * [kube2iam](https://github.com/jtblin/kube2iam): Traffic from Pods to 103 | the AWS Metadata service transits over the unnumbered point-to-point 104 | interface to reach the default namespace before being redirected via 105 | destination NAT. The Pod’s source IP is maintained as kube2iam runs 106 | as a normal Daemon Set. 107 | 108 | ### VPC optimizations 109 | 110 | Our design is heavily optimized for intra-VPC traffic where IPvlan is 111 | the only overhead between the instance’s ethernet interface and the 112 | Pod network namespace. We bias toward traffic remaining within the VPC 113 | and not transiting the IPv4 Internet where veth and NAT overhead is 114 | incurred. Unfortunately, many AWS services require transiting the 115 | Internet; however, both DynamoDB and S3 offer VPC gateway endpoints. 116 | 117 | While we have not yet implemented IPv6 support in our CNI stack, we 118 | have plans to do so in the near future. IPv6 can make use of the 119 | IPvlan interface for both VPC traffic as well as Internet traffic, due 120 | to AWS’s use of public IPv6 addressing within VPCs and support for 121 | egress-only Internet Gateways. NAT and veth overhead will not be 122 | required for this traffic. 123 | 124 | We’re planning to migrate to a VPC endpoint for DynamoDB and use 125 | native IPv6 support for communication to S3. Biasing toward extremely 126 | low overhead IPv6 traffic with higher overhead for IPv4 Internet 127 | traffic is the right future direction. 128 | 129 | # Using with Kubernetes 130 | 131 | ## Supported container runtimes 132 | 133 | `cni-ipvlan-vpc-k8s` is used in production at Lyft with cri-o for 134 | non-GPU workloads and Docker w/ nvidia-docker for GPU workloads. 135 | 136 | Note that for cri-o, `manage_network_ns_lifecycle` *must* be set to 137 | true. 138 | 139 | ## Prerequisites 140 | 141 | 1. By default, we use a secondary (and tertiary, ...) ENI adapter for 142 | all Pod networking. This allows isolation by security groups or 143 | other constraints on the Kubelet control plane. This requires that 144 | the hosts you are running on can attach at least two ENI 145 | adapters. See: 146 | http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html 147 | Most hosts support > 1 adapter, except for some of the smallest 148 | hardware types. 149 | 1. AWS VPC with a recommended minimum number of subnets equal to the 150 | maximum number of attached ENIs. In the normal case of supporting 151 | up to the default 110 Pods per instance, you'll want five subnets 152 | (one for the control plane on the boot ENI and four subnets for the 153 | Pod ENIs). The example configuration uses adapter index 1 onward 154 | for Pods. We recommend creating a secondary IPv4 CIDR block for 155 | Kubernetes deployments within existing VPCs and subnet 156 | appropriately for the number of ENIs. In our primary region, we 157 | divide up our secondary IPv4 CIDR (/16) into 5 /20s per AZ with 3 158 | AZs. Datadog has provided code which removes the restriction on one 159 | subnet per ENI; however, we've yet to test it thoroughly at Lyft. 160 | 1. (Optional) AWS subnets tagged if you want to limit which ones can 161 | be used. 162 | 1. The `kubelet` process _must_ be started with the `--node-ip` option 163 | if you also use `--cloud-provider=aws`. Use the primary IP on 164 | the boot ENI adapter (eth0). 165 | 1. AWS permissions allowing at least these actions on the _Kubelet_ role: 166 | 167 | "ec2:DescribeSubnets" 168 | "ec2:AttachNetworkInterface" 169 | "ec2:AssignPrivateIpAddresses" 170 | "ec2:UnassignPrivateIpAddresses" 171 | "ec2:CreateNetworkInterface" 172 | "ec2:DescribeNetworkInterfaces" 173 | "ec2:DetachNetworkInterface" 174 | "ec2:DeleteNetworkInterface" 175 | "ec2:ModifyNetworkInterfaceAttribute" 176 | "ec2:DescribeInstanceTypes" 177 | "ec2:DescribeVpcs" 178 | "ec2:DescribeVpcPeeringConnections" 179 | 180 | ec2:DescribeVpcs is required for m5 and c5 instances because the AWS metadata 181 | server does not return the secondary CIDR block on these instance types. This 182 | requirement will be removed when the issue is fixed. 183 | 184 | ec2:DescribeVpcPeeringConnections is only required if routeToVpcPeers is 185 | enabled on the plugin. 186 | 187 | See [Security Considerations](#security-considerations) below for more on 188 | the implications of these permissions. 189 | 190 | 191 | ## Building 192 | 193 | cni-ipvlan-vpc-k8s requires `dep` for dependency management. Please see 194 | https://github.com/golang/dep#setup for build instructions. In a 195 | pinch, you may `go get -u github.com/golang/dep/cmd/dep`. 196 | 197 | go get github.com/lyft/cni-ipvlan-vpc-k8s 198 | cd $GOPATH/src/github.com/lyft/cni-ipvlan-vpc-k8s 199 | make build 200 | 201 | ## Example Configuration 202 | 203 | This example CNI conflist creates Pod IPs on the secondary and above 204 | ENI adapters and chains with the upstream ipvlan plugin (0.7.0 or 205 | later required) and the `cni-ipvlan-vpc-k8s-unnumbered-ptp` plugin to 206 | create unnumbered point-to-point links back to the default namespace 207 | from each Pod. New interfaces will be attached to subnets tagged with 208 | `kubernetes_kubelet` = `true`, and created with the defined security 209 | groups. 210 | 211 | Routes are automatically formed for the VPC on the `ipvlan` adapter. 212 | 213 | ipMasq is enabled to use the host-IP for egress to the Internet as 214 | well as providing access to services such as `kube2iam`. `kube2iam` is 215 | not a dependency of this software. 216 | 217 | ``` 218 | { 219 | "cniVersion": "0.3.1", 220 | "name": "cni-ipvlan-vpc-k8s", 221 | "plugins": [ 222 | { 223 | "cniVersion": "0.3.1", 224 | "type": "cni-ipvlan-vpc-k8s-ipam", 225 | "interfaceIndex": 1, 226 | "subnetTags": { 227 | "kubernetes_kubelet": "true" 228 | }, 229 | "secGroupIds": [ 230 | "sg-1234", 231 | "sg-5678" 232 | ] 233 | }, 234 | { 235 | "cniVersion": "0.3.1", 236 | "type": "cni-ipvlan-vpc-k8s-ipvlan", 237 | "mode": "l2" 238 | }, 239 | { 240 | "cniVersion": "0.3.1", 241 | "type": "cni-ipvlan-vpc-k8s-unnumbered-ptp", 242 | "hostInterface": "eth0", 243 | "containerInterface": "eth1", 244 | "ipMasq": true 245 | } 246 | ] 247 | } 248 | ``` 249 | 250 | ### Other configuration flags 251 | 252 | In the above `cni-ipvlan-vpc-k8s-ipam` config, several options are 253 | available: 254 | 255 | - `interfaceIndex`: We also recommend never using the boot ENI 256 | adapter with this plugin (though it is possible). By setting 257 | `interfaceIndex` to 1, the plugin will only allocate IPs (and add 258 | new adapters) starting at `eth1`. 259 | - `subnetTags`: When allocating new adapters, by default the plugin 260 | will use all available subnets within the availability zone. You 261 | can restrict which subnets the plugin will use by specifying key / 262 | value tag names that must be matched in order for the plugin to be 263 | considered. These tags are set via the AWS API or in the AWS 264 | Console on the subnet object. 265 | - `secGroupIds`: When allocating a new ENI adapter, these interface 266 | groups will be assigned to the adapter. Specify the `sg-xxxx` 267 | interface group ID. 268 | - `skipDeallocation`: `true` or `false` - when set to `true`, this 269 | plugin will never remove a secondary IP address from an 270 | adapter. Useful in workloads that churn many pods to reduce the AWS 271 | ratelimits for configuring the VPC (which are low and cannot be 272 | raised above a certain threshold). 273 | - `routeToVpcPeers`: `true` or `false` - When set to `true`, the 274 | plugin will make a (cached) call to `DescribeVpcPeeringConnections` 275 | to enumerate all peered VPCs. Routes will be added so connections 276 | to these VPCs will be sourced from the IPvlan adapter in the pod 277 | and not through the host masquerade. 278 | - `routeToCidrs`: List of CIDRs. Routes will be added so connections 279 | to these CIDRs will be sourced from the IPvlan adapter in the pod 280 | and not through the host masquerade. 281 | - `reuseIPWait`: Seconds to wait before free IP addresses are made 282 | available for reuse by Pods. Defaults to 60 seconds. `reuseIPWait` 283 | functions as both a lock to prevent addresses from being grabbed by 284 | Pods spinning up in between the stages of chained CNI plugin 285 | execution and as a method of delaying when a new Pod can grab the 286 | same IP address of a terminating Pod. 287 | 288 | 289 | ### IP address lifecycle management 290 | 291 | As new Pods are created, if needed, secondary IP addresses are added 292 | to secondary ENI adapters until they reach capacity. A lightweight 293 | file-based registry stores hints containing free IP addresses 294 | available to the instance to prevent unnecessary churn from adding and 295 | removing IPs to and from ENI adapters, which is a fairly heavyweight 296 | AWS process. By default, free IP addresses are made available for 297 | reuse by Pods after being unused for at least 60 seconds. To handle 298 | cases where IPs are not frequently reused by Pods, and an excess of 299 | free IP addresses becomes available on an instance, a systemd timer is 300 | recommended to garbage collect these old IPs. 301 | 302 | Sample cni-gc.service: 303 | ```[Unit] 304 | Description=Garbage collect IPs unused for 15 minutes 305 | 306 | [Service] 307 | Type=oneshot 308 | ExecStart=/usr/local/bin/cni-ipvlan-vpc-k8s-tool registry-gc --free-after=15m 309 | ``` 310 | 311 | Sample cni-gc.timer: 312 | ``` 313 | [Unit] 314 | Description=Run cni-gc every 5 minutes 315 | 316 | [Timer] 317 | OnBootSec=5min 318 | OnUnitActiveSec=5min 319 | 320 | [Install] 321 | WantedBy=timers.target 322 | ``` 323 | 324 | ## The CLI Tool 325 | 326 | This plugin ships a CLI tool which can be useful to inspect the state 327 | of the system or perform certain actions (such as provisioning an 328 | adapter at instance cloud-init time). 329 | 330 | Run `cni-ipvlan-vpc-k8s-tool --help` for a complete listing of 331 | options. 332 | 333 | NAME: 334 | cni-ipvlan-vpc-k8s-tool - Interface with ENI adapters and CNI bindings for those 335 | 336 | USAGE: 337 | cni-ipvlan-vpc-k8s-tool [global options] command [command options] [arguments...] 338 | 339 | VERSION: 340 | v-next 341 | 342 | COMMANDS: 343 | new-interface Create a new interface 344 | remove-interface Remove an existing interface 345 | deallocate Deallocate a private IP 346 | allocate-first-available Allocate a private IP on the first available interface 347 | free-ips List all currently unassigned AWS IP addresses 348 | eniif List all ENI interfaces and their setup with addresses 349 | addr List all bound IP addresses 350 | subnets Show available subnets for this host 351 | limits Display limits for ENI for this instance type 352 | bugs Show any bugs associated with this instance 353 | vpccidr Show the VPC CIDRs associated with current interfaces 354 | vpcpeercidr Show the peered VPC CIDRs associated with current interfaces 355 | registry-list List all known free IPs in the internal registry 356 | registry-gc Free all IPs that have remained unused for a given time interval 357 | help, h Shows a list of commands or help for one command 358 | 359 | GLOBAL OPTIONS: 360 | --help, -h show help 361 | --version, -v print the version 362 | 363 | COPYRIGHT: 364 | (c) 2017-2018 Lyft Inc. 365 | 366 | 367 | ## Security Considerations 368 | 369 | In Kubernetes, pods and kubelets are assumed to have static IP addresses that 370 | are assigned for the lifetime of the object. However, the EC2 IAM permissions 371 | required by `cni-ipvlan-vpc-k8s` enable authorized principals to manipulate 372 | network interfaces and IP addresses, which could be used to remap IP addresses 373 | and "take over" the IP address of an existing pod or kubelet. Such an IP 374 | address takeover could allow impersonation of a pod or kubelet at the network 375 | layer, and disrupt the availability of your Kubernetes cluster. 376 | 377 | IP address takeovers are possible in the following situations: 378 | * Compromise of a kubelet instance configured to run `cni-ipvlan-vpc-k8s` with 379 | the required IAM permissions. 380 | * Use (or abuse) of the EC2 ENI and IP Address manipulation APIs by a user or 381 | service in your AWS account authorized to do so. 382 | 383 | Consider taking the following actions to reduce the likelihood and impact of IP 384 | takeover attacks: 385 | * Limit the number of principals authorized to manipulate ENIs and IP 386 | addresses. 387 | * Do not rely exclusively on the Kubernetes control plane to ensure you're 388 | connected to the pod you expect. Deploy mutual TLS (mTLS) or other end-to-end 389 | authentication to authenticate clients and pods at the application layer. 390 | 391 | # Get Support: Mailing Lists and Chat 392 | 393 | * Announcement list - new releases will be announced here: 394 | https://groups.google.com/forum/#!forum/cni-ipvlan-vpc-k8s-announce 395 | * Users discussion list: 396 | https://groups.google.com/forum/#!forum/cni-ipvlan-vpc-k8s-users 397 | * Gitter discussion: https://gitter.im/lyft/cni-ipvlan-vpc-k8s 398 | -------------------------------------------------------------------------------- /aws/allocate.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "sort" 8 | "time" 9 | 10 | "github.com/aws/aws-sdk-go/service/ec2" 11 | ) 12 | 13 | // AllocationResult contains a net.IP / Interface pair 14 | type AllocationResult struct { 15 | *net.IP 16 | Interface Interface 17 | } 18 | 19 | // AllocateClient offers IP allocation on interfaces 20 | type AllocateClient interface { 21 | AllocateIPsOn(intf Interface, batchSize int64) ([]*AllocationResult, error) 22 | AllocateIPsFirstAvailableAtIndex(index int, batchSize int64) ([]*AllocationResult, error) 23 | AllocateIPsFirstAvailable(batchSize int64) ([]*AllocationResult, error) 24 | DeallocateIP(ipToRelease *net.IP) error 25 | } 26 | 27 | type allocateClient struct { 28 | aws *awsclient 29 | subnet SubnetsClient 30 | } 31 | 32 | // AllocateIPsOn allocates IPs on a specific interface. 33 | func (c *allocateClient) AllocateIPsOn(intf Interface, batchSize int64) ([]*AllocationResult, error) { 34 | var allocationResults []*AllocationResult 35 | client, err := c.aws.newEC2() 36 | if err != nil { 37 | return nil, err 38 | } 39 | request := ec2.AssignPrivateIpAddressesInput{ 40 | NetworkInterfaceId: &intf.ID, 41 | } 42 | 43 | limits, err := c.aws.ENILimits() 44 | if err != nil { 45 | log.Printf("unable to determine AWS limits, using fallback %v", err) 46 | } 47 | available := limits.IPv4 - int64(len(intf.IPv4s)) 48 | 49 | // If there are fewer IPs left than the batch size, request all the remaining IPs 50 | // batch size 0 conventionally means "request the limit" 51 | if batchSize == 0 || available < batchSize { 52 | batchSize = available 53 | } 54 | 55 | request.SetSecondaryPrivateIpAddressCount(batchSize) 56 | 57 | _, err = client.AssignPrivateIpAddresses(&request) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | registry := &Registry{} 63 | for attempts := 10; attempts > 0; attempts-- { 64 | newIntf, err := c.aws.getInterface(intf.Mac) 65 | if err != nil { 66 | time.Sleep(1.0 * time.Second) 67 | continue 68 | } 69 | 70 | if len(newIntf.IPv4s) != len(intf.IPv4s) { 71 | // New address detected 72 | for _, newip := range newIntf.IPv4s { 73 | found := false 74 | for _, oldip := range intf.IPv4s { 75 | if newip.Equal(oldip) { 76 | found = true 77 | } 78 | } 79 | if !found { 80 | // only return IPs that haven't been previously registered 81 | if exists, err := registry.HasIP(newip); err == nil && !exists { 82 | ipcopy := newip // Need to copy 83 | allocationResult := &AllocationResult{ 84 | &ipcopy, 85 | newIntf, 86 | } 87 | allocationResults = append(allocationResults, allocationResult) 88 | 89 | // New IP. Timestamp the addition as a free IP. 90 | err = registry.TrackIP(newip) 91 | if err != nil { 92 | return allocationResults, fmt.Errorf("failed to track ip: %s", err) 93 | } 94 | } 95 | } 96 | } 97 | if len(allocationResults) > 0 { 98 | return allocationResults, nil 99 | } 100 | } 101 | time.Sleep(1.0 * time.Second) 102 | } 103 | 104 | return nil, fmt.Errorf("Can't locate new IP address from AWS") 105 | } 106 | 107 | // AllocateIPsFirstAvailableAtIndex allocates IP addresses, skipping any adapter < the given index 108 | // Returns a reference to the interface the IPs were allocated on 109 | func (c *allocateClient) AllocateIPsFirstAvailableAtIndex(index int, batchSize int64) ([]*AllocationResult, error) { 110 | interfaces, err := c.aws.GetInterfaces() 111 | if err != nil { 112 | return nil, err 113 | } 114 | limits, err := c.aws.ENILimits() 115 | if err != nil { 116 | log.Printf("unable to determine AWS limits, using fallback %v", err) 117 | } 118 | 119 | var candidates []Interface 120 | for _, intf := range interfaces { 121 | if intf.Number < index { 122 | continue 123 | } 124 | if int64(len(intf.IPv4s)) < limits.IPv4 { 125 | candidates = append(candidates, intf) 126 | } 127 | } 128 | 129 | subnets, err := c.subnet.GetSubnetsForInstance() 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | sort.Sort(SubnetsByAvailableAddressCount(subnets)) 135 | for _, subnet := range subnets { 136 | if subnet.AvailableAddressCount <= 0 { 137 | continue 138 | } 139 | for _, intf := range candidates { 140 | if intf.SubnetID == subnet.ID { 141 | return c.AllocateIPsOn(intf, batchSize) 142 | } 143 | } 144 | } 145 | 146 | return nil, fmt.Errorf("Unable to allocate - no IPs available on any interfaces") 147 | } 148 | 149 | // AllocateIPsFirstAvailable allocates IP addresses on the first available IP address 150 | // Returns a reference to the interface the IPs were allocated on 151 | func (c *allocateClient) AllocateIPsFirstAvailable(batchSize int64) ([]*AllocationResult, error) { 152 | return c.AllocateIPsFirstAvailableAtIndex(0, batchSize) 153 | } 154 | 155 | // DeallocateIP releases an IP back to AWS 156 | func (c *allocateClient) DeallocateIP(ipToRelease *net.IP) error { 157 | client, err := c.aws.newEC2() 158 | if err != nil { 159 | return err 160 | } 161 | interfaces, err := c.aws.GetInterfaces() 162 | if err != nil { 163 | return err 164 | } 165 | for _, intf := range interfaces { 166 | for _, ip := range intf.IPv4s { 167 | if ipToRelease.Equal(ip) { 168 | request := ec2.UnassignPrivateIpAddressesInput{} 169 | request.SetNetworkInterfaceId(intf.ID) 170 | strIP := ipToRelease.String() 171 | request.SetPrivateIpAddresses([]*string{&strIP}) 172 | _, err = client.UnassignPrivateIpAddresses(&request) 173 | return err 174 | } 175 | } 176 | } 177 | 178 | return fmt.Errorf("IP not found - can't release") 179 | } 180 | -------------------------------------------------------------------------------- /aws/bugs.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // Bug defines a bug name and a function to check if the 8 | // system is affected 9 | type Bug struct { 10 | Name string 11 | HasBug func() bool 12 | } 13 | 14 | // ListBugs returns an enumerated set of all known bugs in AWS or instances 15 | // that this instance is afflicted by 16 | func ListBugs(meta MetadataClient) []Bug { 17 | bugs := []Bug{ 18 | { 19 | Name: "broken_cidr", 20 | HasBug: func() bool { return HasBugBrokenVPCCidrs(meta) }, 21 | }, 22 | } 23 | return bugs 24 | } 25 | 26 | // HasBugBrokenVPCCidrs returns true if this instance has a known defective 27 | // meta-data service which will not return secondary VPC CIDRs. As of January 2018, 28 | // this covers c5 and m5 class hardware. 29 | func HasBugBrokenVPCCidrs(meta MetadataClient) bool { 30 | itype := meta.InstanceType() 31 | family := strings.Split(itype, ".")[0] 32 | switch family { 33 | case "c5", "m5", "c5d", "m5d", "m5a", "r5", "r5d", "r5a": 34 | return true 35 | default: 36 | return false 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /aws/cache/cacheable.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "path" 8 | "time" 9 | 10 | "github.com/lyft/cni-ipvlan-vpc-k8s/lib" 11 | ) 12 | 13 | const ( 14 | cacheRoot = "/run" 15 | cacheProgram = "cni-ipvlan-vpc-k8s" 16 | ) 17 | 18 | // State defines the return of the Store and Get calls 19 | type State int 20 | 21 | const ( 22 | // CacheFound means the key was found and valid 23 | CacheFound State = iota 24 | // CacheExpired means the key was found, but has expired. The value returned is not valid. 25 | CacheExpired 26 | // CacheNoEntry means the key was not found. 27 | CacheNoEntry 28 | // CacheNotAvailable means the cache system is not working as expected and has an internal error 29 | CacheNotAvailable 30 | ) 31 | 32 | func cachePath() string { 33 | uid := os.Getuid() 34 | if uid != 0 { 35 | return path.Join(cacheRoot, "user", fmt.Sprintf("%d", os.Getuid()), cacheProgram) 36 | } 37 | 38 | return path.Join(cacheRoot, cacheProgram) 39 | } 40 | 41 | // Cacheable defines metadata for objects which can be cached to files as JSON 42 | type Cacheable struct { 43 | Expires lib.JSONTime `json:"_expires"` 44 | Contents interface{} `json:"contents"` 45 | } 46 | 47 | func ensureDirectory() error { 48 | cachePath := cachePath() 49 | info, err := os.Stat(cachePath) 50 | if err == nil && info.IsDir() { 51 | return nil 52 | } 53 | 54 | err = os.MkdirAll(cachePath, os.ModeDir|0700) 55 | return err 56 | } 57 | 58 | // Get gets a key from the named cache file 59 | func Get(key string, decodeTo interface{}) State { 60 | err := ensureDirectory() 61 | if err != nil { 62 | return CacheNotAvailable 63 | } 64 | 65 | file, err := os.Open(path.Join(cachePath(), key)) 66 | if err != nil { 67 | return CacheNoEntry 68 | } 69 | 70 | defer file.Close() 71 | 72 | var contents Cacheable 73 | contents.Contents = decodeTo 74 | decoder := json.NewDecoder(file) 75 | err = decoder.Decode(&contents) 76 | if err != nil { 77 | return CacheNoEntry 78 | } 79 | 80 | if contents.Expires.Time.Before(time.Now()) { 81 | return CacheExpired 82 | } 83 | 84 | return CacheFound 85 | } 86 | 87 | // Store stores the given data interface as a JSON file with a given expiration time 88 | // under the given key. 89 | func Store(key string, lifetime time.Duration, data interface{}) State { 90 | err := ensureDirectory() 91 | if err != nil { 92 | return CacheNotAvailable 93 | } 94 | 95 | key = path.Join(cachePath(), key) 96 | 97 | file, err := os.OpenFile(key, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) 98 | if err != nil { 99 | return CacheNotAvailable 100 | } 101 | defer file.Close() 102 | 103 | encoder := json.NewEncoder(file) 104 | if encoder == nil { 105 | return CacheNotAvailable 106 | } 107 | 108 | var contents Cacheable 109 | contents.Expires.Time = time.Now().Add(lifetime) 110 | contents.Contents = data 111 | err = encoder.Encode(&contents) 112 | if err != nil { 113 | return CacheNotAvailable 114 | } 115 | 116 | return CacheFound 117 | } 118 | -------------------------------------------------------------------------------- /aws/cache/cacheable_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | type Thing struct { 9 | Value string `json:"value"` 10 | TValue int 11 | } 12 | 13 | func TestGet(t *testing.T) { 14 | var d Thing 15 | state := Get("key_not_exist", &d) 16 | if state != CacheNoEntry { 17 | t.Errorf("Empty cache did not return a valid state %v", state) 18 | } 19 | 20 | d.Value = "Hello" 21 | d.TValue = 12 22 | 23 | state = Store("hello", 30*time.Second, &d) 24 | if state != CacheFound { 25 | t.Errorf("Invalid store of the cache key %v", state) 26 | } 27 | 28 | var e Thing 29 | state = Get("hello", &e) 30 | if state != CacheFound { 31 | t.Errorf("Can't reload existing key %v", state) 32 | } 33 | 34 | if d.Value != e.Value && d.TValue != e.TValue { 35 | t.Errorf("%v != %v", d, t) 36 | } 37 | 38 | // Test expiration 39 | Store("hello2", 1*time.Millisecond, &d) 40 | time.Sleep(100 * time.Millisecond) 41 | state = Get("hello2", &e) 42 | if state != CacheExpired { 43 | t.Error("Cache did not expire") 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /aws/client.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/ec2" 11 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 12 | ) 13 | 14 | type awsclient struct { 15 | sess *session.Session 16 | metaData *ec2metadata.EC2Metadata 17 | 18 | idDoc *ec2metadata.EC2InstanceIdentityDocument 19 | onceIDDoc sync.Once 20 | 21 | ec2Client ec2iface.EC2API 22 | onceEc2 sync.Once 23 | } 24 | 25 | type combinedClient struct { 26 | *subnetsCacheClient 27 | *awsclient 28 | *interfaceClient 29 | *allocateClient 30 | *vpcCacheClient 31 | } 32 | 33 | // Client offers all of the supporting AWS services 34 | type Client interface { 35 | InterfaceClient 36 | LimitsClient 37 | MetadataClient 38 | SubnetsClient 39 | AllocateClient 40 | VPCClient 41 | } 42 | 43 | var defaultClient *combinedClient 44 | 45 | // DefaultClient that is setup with known defaults 46 | var DefaultClient Client 47 | 48 | func init() { 49 | awsClient := &awsclient{} 50 | subnets := &subnetsCacheClient{ 51 | &subnetsClient{aws: awsClient}, 52 | 5 * time.Minute, 53 | } 54 | defaultClient = &combinedClient{ 55 | subnets, 56 | awsClient, 57 | &interfaceClient{awsClient, subnets}, 58 | &allocateClient{awsClient, subnets}, 59 | &vpcCacheClient{ 60 | &vpcclient{awsClient}, 61 | 1 * time.Hour, 62 | }, 63 | } 64 | 65 | DefaultClient = defaultClient 66 | defaultClient.sess = session.Must(session.NewSession()) 67 | defaultClient.metaData = ec2metadata.New(defaultClient.sess) 68 | } 69 | 70 | func (c *awsclient) getIDDoc() (*ec2metadata.EC2InstanceIdentityDocument, error) { 71 | var err error 72 | c.onceIDDoc.Do(func() { 73 | // Allow mock ID documents to be inserted 74 | if c.idDoc == nil { 75 | var instance ec2metadata.EC2InstanceIdentityDocument 76 | instance, err = c.metaData.GetInstanceIdentityDocument() 77 | if err != nil { 78 | return 79 | } 80 | // Cache the document 81 | c.idDoc = &instance 82 | } 83 | }) 84 | return c.idDoc, err 85 | } 86 | 87 | // Allocate a new EC2 client configured for the current instance 88 | // region. Clients are re-used across multiple calls 89 | func (c *awsclient) newEC2() (ec2iface.EC2API, error) { 90 | var err error 91 | c.onceEc2.Do(func() { 92 | var id *ec2metadata.EC2InstanceIdentityDocument 93 | id, err = c.getIDDoc() 94 | if err != nil { 95 | return 96 | } 97 | if c.ec2Client == nil { 98 | // Use the sess object already defined 99 | c.ec2Client = ec2.New(c.sess, aws.NewConfig().WithRegion(id.Region)) 100 | } 101 | }) 102 | return c.ec2Client, err 103 | } 104 | -------------------------------------------------------------------------------- /aws/client_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 7 | ) 8 | 9 | func TestClientCreate(t *testing.T) { 10 | oldIDDoc := defaultClient.idDoc 11 | defer func() { defaultClient.idDoc = oldIDDoc }() 12 | 13 | defaultClient.idDoc = &ec2metadata.EC2InstanceIdentityDocument{ 14 | Region: "us-east-1", 15 | AvailabilityZone: "us-east-1a", 16 | } 17 | 18 | client, err := defaultClient.newEC2() 19 | if err != nil { 20 | t.Errorf("Error generated %v", err) 21 | } 22 | 23 | if client == nil { 24 | t.Errorf("No client returned %v", err) 25 | } 26 | 27 | client2, err := defaultClient.newEC2() 28 | if client != client2 || err != nil { 29 | t.Errorf("Clients returned were not identical (no caching)") 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /aws/freeip.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "github.com/lyft/cni-ipvlan-vpc-k8s/nl" 5 | ) 6 | 7 | // FindFreeIPsAtIndex locates free IP addresses by comparing the assigned list 8 | // from the EC2 metadata service and the currently used addresses 9 | // within netlink. This is inherently somewhat racey - for example 10 | // newly provisioned addresses may not show up immediately in metadata 11 | // and are subject to a few seconds of delay. 12 | func FindFreeIPsAtIndex(index int, updateRegistry bool) ([]*AllocationResult, error) { 13 | freeIps := []*AllocationResult{} 14 | registry := &Registry{} 15 | 16 | interfaces, err := DefaultClient.GetInterfaces() 17 | if err != nil { 18 | return nil, err 19 | } 20 | assigned, err := nl.GetIPs() 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | for _, intf := range interfaces { 26 | if intf.Number < index { 27 | continue 28 | } 29 | for _, intfIP := range intf.IPv4s { 30 | found := false 31 | for _, assignedIP := range assigned { 32 | if assignedIP.IPNet.IP.Equal(intfIP) { 33 | found = true 34 | break 35 | } 36 | } 37 | if !found { 38 | intfIPCopy := intfIP 39 | // No match, record as free 40 | freeIps = append(freeIps, &AllocationResult{ 41 | &intfIPCopy, 42 | intf, 43 | }) 44 | } 45 | if updateRegistry { 46 | if exists, err := registry.HasIP(intfIP); err == nil && !exists && !found { 47 | // track IP as free if it hasn't been registered before 48 | err = registry.TrackIP(intfIP) 49 | if err != nil { 50 | return freeIps, err 51 | } 52 | } else if found { 53 | // mark IP as in use 54 | err = registry.ForgetIP(intfIP) 55 | if err != nil { 56 | return freeIps, err 57 | } 58 | } 59 | } 60 | } 61 | } 62 | 63 | return freeIps, nil 64 | } 65 | -------------------------------------------------------------------------------- /aws/interface.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "os" 8 | "sort" 9 | "time" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/service/ec2" 13 | 14 | "github.com/lyft/cni-ipvlan-vpc-k8s/nl" 15 | ) 16 | 17 | var ( 18 | interfacePollWaitTime = 1000 * time.Millisecond 19 | interfaceSettleTime = 30 * time.Second 20 | interfaceDetachWaitTime = 1 * time.Second 21 | interfacePostDetachSettleTime = 5 * time.Second 22 | interfaceDetachAttempts = 20 // interfaceDetachAttempts * interfaceDetachWaitTime = total wait time 23 | ) 24 | 25 | // InterfaceClient provides methods for allocating and deallocating interfaces 26 | type InterfaceClient interface { 27 | NewInterfaceOnSubnetAtIndex(index int, secGrps []string, subnet Subnet, ipBatchSize int64) (*Interface, error) 28 | NewInterface(secGrps []string, requiredTags map[string]string, ipBatchSize int64) (*Interface, error) 29 | RemoveInterface(interfaceIDs []string) error 30 | } 31 | 32 | type interfaceClient struct { 33 | aws *awsclient 34 | subnet SubnetsClient 35 | } 36 | 37 | // NewInterfaceOnSubnetAtIndex creates a new Interface with a specified subnet and index 38 | func (c *interfaceClient) NewInterfaceOnSubnetAtIndex(index int, secGrps []string, subnet Subnet, ipBatchSize int64) (*Interface, error) { 39 | client, err := c.aws.newEC2() 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | idDoc, err := c.aws.getIDDoc() 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | createReq := &ec2.CreateNetworkInterfaceInput{} 50 | createReq.SetDescription(fmt.Sprintf("CNI-ENI %v", idDoc.InstanceID)) 51 | secGrpsPtr := []*string{} 52 | for _, grp := range secGrps { 53 | newgrp := grp // Need to copy 54 | secGrpsPtr = append(secGrpsPtr, &newgrp) 55 | } 56 | 57 | createReq.SetGroups(secGrpsPtr) 58 | createReq.SetSubnetId(subnet.ID) 59 | 60 | // Subtract 1 to Account for primary IP 61 | limits, err := c.aws.ENILimits() 62 | if err != nil { 63 | log.Printf("unable to determine AWS limits, using fallback %v", err) 64 | } 65 | 66 | // batch size 0 conventionally means "request the limit" 67 | if ipBatchSize == 0 || ipBatchSize > limits.IPv4 { 68 | ipBatchSize = limits.IPv4 69 | } 70 | 71 | // We will already get a primary IP on the ENI 72 | ipBatchSize = ipBatchSize - 1 73 | if ipBatchSize > 0 { 74 | createReq.SecondaryPrivateIpAddressCount = &ipBatchSize 75 | } 76 | 77 | resp, err := client.CreateNetworkInterface(createReq) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | // resp.NetworkInterface.NetworkInterfaceId 83 | attachReq := &ec2.AttachNetworkInterfaceInput{} 84 | attachReq.SetDeviceIndex(int64(index)) 85 | attachReq.SetInstanceId(idDoc.InstanceID) 86 | attachReq.SetNetworkInterfaceId(*resp.NetworkInterface.NetworkInterfaceId) 87 | 88 | attachResp, err := client.AttachNetworkInterface(attachReq) 89 | if err != nil { 90 | // We attempt to remove the interface we just made due to attachment failure 91 | delReq := &ec2.DeleteNetworkInterfaceInput{} 92 | delReq.SetNetworkInterfaceId(*resp.NetworkInterface.NetworkInterfaceId) 93 | 94 | _, delErr := client.DeleteNetworkInterface(delReq) 95 | if delErr != nil { 96 | return nil, delErr 97 | } 98 | return nil, err 99 | } 100 | 101 | // We have an attachment ID from the last API, which lets us mark the 102 | // interface as delete on termination 103 | changes := &ec2.NetworkInterfaceAttachmentChanges{} 104 | changes.SetAttachmentId(*attachResp.AttachmentId) 105 | changes.SetDeleteOnTermination(true) 106 | modifyReq := &ec2.ModifyNetworkInterfaceAttributeInput{} 107 | modifyReq.SetAttachment(changes) 108 | modifyReq.SetNetworkInterfaceId(*resp.NetworkInterface.NetworkInterfaceId) 109 | 110 | _, err = client.ModifyNetworkInterfaceAttribute(modifyReq) 111 | if err != nil { 112 | // Continue anyway 113 | fmt.Fprintf(os.Stderr, 114 | "Unable to mark interface for deletion due to %v", 115 | err) 116 | } 117 | 118 | for start := time.Now(); time.Since(start) <= interfaceSettleTime; time.Sleep(interfacePollWaitTime) { 119 | newInterfaces, err := c.aws.GetInterfaces() 120 | if err != nil { 121 | // The metadata server is inconsistent - for example, not 122 | // all of the nodes under the interface will populate at once 123 | // and instead return a 404 error. We just swallow this error here and 124 | // continue on. 125 | continue 126 | } 127 | for i, intf := range newInterfaces { 128 | if intf.Mac == *resp.NetworkInterface.MacAddress { 129 | registry := &Registry{} 130 | // Timestamp the addition of all the new IPs in the registry. 131 | for _, privateIPAddress := range resp.NetworkInterface.PrivateIpAddresses { 132 | if privateIPAddr := net.ParseIP(*privateIPAddress.PrivateIpAddress); privateIPAddr != nil { 133 | _ = registry.TrackIPAtEpoch(privateIPAddr) 134 | } 135 | } 136 | // Interfaces are sorted by device number. The first one is the main one 137 | mainIf := newInterfaces[0].IfName 138 | configureInterface(&newInterfaces[i], mainIf) 139 | return &newInterfaces[i], nil 140 | } 141 | } 142 | 143 | } 144 | 145 | return nil, fmt.Errorf("interface did not attach in time") 146 | } 147 | 148 | // Fire and forget method to configure an interface 149 | func configureInterface(intf *Interface, mainIf string) { 150 | // Found a match, going to try to make sure the interface is up 151 | err := nl.UpInterfacePoll(intf.LocalName()) 152 | if err != nil { 153 | fmt.Fprintf(os.Stderr, 154 | "Interface %v could not be enabled. Networking will be broken.\n", 155 | intf.LocalName()) 156 | return 157 | } 158 | baseMtu, err := nl.GetMtu(mainIf) 159 | if err != nil || baseMtu < 1000 || baseMtu > 9001 { 160 | return 161 | } 162 | err = nl.SetMtu(intf.LocalName(), baseMtu) 163 | if err != nil { 164 | fmt.Fprintf(os.Stderr, "failed to configure mtu: %s", err) 165 | } 166 | } 167 | 168 | // NewInterface creates an Interface based on specified parameters 169 | func (c *interfaceClient) NewInterface(secGrps []string, requiredTags map[string]string, ipBatchSize int64) (*Interface, error) { 170 | subnets, err := c.subnet.GetSubnetsForInstance() 171 | if err != nil { 172 | return nil, err 173 | } 174 | 175 | existingInterfaces, err := c.aws.GetInterfaces() 176 | if err != nil { 177 | return nil, err 178 | } 179 | 180 | limits, err := c.aws.ENILimits() 181 | if err != nil { 182 | log.Printf("unable to determine AWS limits, using fallback %v", err) 183 | } 184 | if int64(len(existingInterfaces)) >= limits.Adapters { 185 | return nil, fmt.Errorf("too many adapters on this instance already") 186 | } 187 | 188 | var availableSubnets []Subnet 189 | 190 | OUTER: 191 | for _, newSubnet := range subnets { 192 | // Match incoming tags 193 | for tagKey, tagValue := range requiredTags { 194 | value, ok := newSubnet.Tags[tagKey] 195 | // Skip untagged subnets and ones not matching 196 | // the required tag 197 | if !ok || (ok && value != tagValue) { 198 | continue OUTER 199 | } 200 | } 201 | availableSubnets = append(availableSubnets, newSubnet) 202 | } 203 | 204 | // assign new interfaces to subnets with most available addresses 205 | sort.Sort(SubnetsByAvailableAddressCount(availableSubnets)) 206 | 207 | if len(availableSubnets) <= 0 { 208 | return nil, fmt.Errorf("No subnets are available which haven't already been used") 209 | } 210 | 211 | return c.NewInterfaceOnSubnetAtIndex(len(existingInterfaces), secGrps, availableSubnets[0], ipBatchSize) 212 | } 213 | 214 | // RemoveInterface graceful shutdown and removal of interfaces 215 | // Simply detach the interface, wait for it to come down and then 216 | // removes. 217 | func (c *awsclient) RemoveInterface(interfaceIDs []string) error { 218 | client, err := c.newEC2() 219 | if err != nil { 220 | return err 221 | } 222 | 223 | for _, interfaceID := range interfaceIDs { 224 | // TODO: check if there is any other interface on this namespace? 225 | 226 | // We need the interface AttachmentId to detach 227 | interfaceDescription, err := c.describeNetworkInterface(interfaceID) 228 | if err != nil { 229 | return err 230 | } 231 | 232 | detachInterfaceInput := &ec2.DetachNetworkInterfaceInput{ 233 | AttachmentId: interfaceDescription.Attachment.AttachmentId, 234 | DryRun: aws.Bool(false), 235 | Force: aws.Bool(false), 236 | } 237 | 238 | // Detach the networkinterface 239 | _, err = client.DetachNetworkInterface(detachInterfaceInput) 240 | if err != nil { 241 | fmt.Printf("Error occurced when trying to detach %v interface, use --force to override this check", interfaceID) 242 | return err 243 | } 244 | 245 | // Wait for the interface to be removed 246 | if err := c.waitUtilInterfaceDetaches(interfaceID); err != nil { 247 | return err 248 | } 249 | 250 | // Even after the interface detaches, you cannot delete right away 251 | time.Sleep(interfacePostDetachSettleTime) 252 | 253 | // Now we can safely remove the interface 254 | if err := c.deleteInterface(interfaceID); err != nil { 255 | return err 256 | } 257 | } 258 | return nil 259 | } 260 | 261 | func (c *awsclient) deleteInterface(interfaceID string) error { 262 | client, err := c.newEC2() 263 | if err != nil { 264 | return err 265 | } 266 | 267 | deleteInterfaceInput := &ec2.DeleteNetworkInterfaceInput{ 268 | NetworkInterfaceId: aws.String(interfaceID), 269 | } 270 | 271 | _, err = client.DeleteNetworkInterface(deleteInterfaceInput) 272 | return err 273 | } 274 | 275 | func (c *awsclient) waitUtilInterfaceDetaches(interfaceID string) error { 276 | var interfaceDescription *ec2.NetworkInterface 277 | 278 | interfaceDescription, err := c.describeNetworkInterface(interfaceID) 279 | if err != nil { 280 | return err 281 | } 282 | 283 | // Once the ENI is in available state, we are ok to delete it 284 | for attempt := 0; *interfaceDescription.Status != "available"; attempt++ { 285 | interfaceDescription, err = c.describeNetworkInterface(interfaceID) 286 | if err != nil { 287 | return err 288 | } 289 | 290 | if attempt == interfaceDetachAttempts { 291 | return fmt.Errorf("Interface %v has not detached yet, use --force to override this check", interfaceID) 292 | } 293 | 294 | time.Sleep(interfaceDetachWaitTime) 295 | } 296 | 297 | return nil 298 | } 299 | 300 | func (c *awsclient) describeNetworkInterface(interfaceID string) (*ec2.NetworkInterface, error) { 301 | client, err := c.newEC2() 302 | if err != nil { 303 | return nil, err 304 | } 305 | 306 | interfaceIDList := []string{interfaceID} 307 | describeInterfaceInput := &ec2.DescribeNetworkInterfacesInput{ 308 | NetworkInterfaceIds: aws.StringSlice(interfaceIDList), 309 | } 310 | 311 | interfaceDescribeOutput, err := client.DescribeNetworkInterfaces(describeInterfaceInput) 312 | if err != nil { 313 | return nil, err 314 | } 315 | 316 | if len(interfaceDescribeOutput.NetworkInterfaces) <= 0 { 317 | return nil, fmt.Errorf("Cannot describe interface, it might not exist") 318 | } 319 | 320 | return interfaceDescribeOutput.NetworkInterfaces[0], nil 321 | } 322 | -------------------------------------------------------------------------------- /aws/interface_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/ec2" 9 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 10 | ) 11 | 12 | type ec2ClientMock struct { 13 | ec2iface.EC2API 14 | NetworkDescribeResponse ec2.DescribeNetworkInterfacesOutput 15 | NetworkDeleteResponse ec2.DeleteNetworkInterfaceOutput 16 | NetworkDetachResponse ec2.DetachNetworkInterfaceOutput 17 | } 18 | 19 | func (e *ec2ClientMock) DescribeNetworkInterfaces(in *ec2.DescribeNetworkInterfacesInput) (*ec2.DescribeNetworkInterfacesOutput, error) { 20 | return &e.NetworkDescribeResponse, nil 21 | } 22 | 23 | func (e *ec2ClientMock) DeleteNetworkInterface(in *ec2.DeleteNetworkInterfaceInput) (*ec2.DeleteNetworkInterfaceOutput, error) { 24 | return &e.NetworkDeleteResponse, nil 25 | } 26 | 27 | func (e *ec2ClientMock) DetachNetworkInterface(in *ec2.DetachNetworkInterfaceInput) (*ec2.DetachNetworkInterfaceOutput, error) { 28 | return &e.NetworkDetachResponse, nil 29 | } 30 | 31 | // func TestNewInterfaceOnSubnetAtIndex(t *testing.T) {} 32 | // func TestConfigureInterface(t *testing.T) {} 33 | // func TestNewInterface(t *testing.T) {} 34 | 35 | func TestRemoveInterface(t *testing.T) { 36 | interfaceDetachAttempts = 1 37 | interfacePostDetachSettleTime = 1 38 | 39 | cases := []struct { 40 | Input []string 41 | NetworkDescribeResponse ec2.DescribeNetworkInterfacesOutput 42 | }{ 43 | { 44 | Input: []string{"eni-lyft-1"}, 45 | NetworkDescribeResponse: ec2.DescribeNetworkInterfacesOutput{ 46 | NetworkInterfaces: []*ec2.NetworkInterface{ 47 | { 48 | Attachment: &ec2.NetworkInterfaceAttachment{ 49 | AttachmentId: aws.String("eni-lyft-1-attachmentid"), 50 | }, 51 | NetworkInterfaceId: aws.String("eni-lyft-1"), 52 | Status: aws.String("available"), 53 | }, 54 | }, 55 | }, 56 | }, 57 | { 58 | Input: []string{"eni-lyft-1", "eni-lyft-2"}, 59 | NetworkDescribeResponse: ec2.DescribeNetworkInterfacesOutput{ 60 | NetworkInterfaces: []*ec2.NetworkInterface{ 61 | { 62 | Attachment: &ec2.NetworkInterfaceAttachment{ 63 | AttachmentId: aws.String("eni-lyft-1-attachmentid"), 64 | }, 65 | NetworkInterfaceId: aws.String("eni-lyft-1"), 66 | Status: aws.String("available"), 67 | }, 68 | { 69 | Attachment: &ec2.NetworkInterfaceAttachment{ 70 | AttachmentId: aws.String("eni-lyft-2-attachmentid"), 71 | }, 72 | NetworkInterfaceId: aws.String("eni-lyft-2"), 73 | Status: aws.String("pending"), 74 | }, 75 | }, 76 | }, 77 | }, 78 | } 79 | 80 | for i, c := range cases { 81 | defaultClient.ec2Client = &ec2ClientMock{ 82 | NetworkDescribeResponse: c.NetworkDescribeResponse, 83 | } 84 | err := defaultClient.RemoveInterface(c.Input) 85 | 86 | if err != nil { 87 | t.Fatalf("%d Mock returned an error: %v", i, err) 88 | } 89 | } 90 | } 91 | 92 | func TestDeleteInterface(t *testing.T) { 93 | cases := []struct { 94 | Input string 95 | Response ec2.DeleteNetworkInterfaceOutput 96 | }{ 97 | { 98 | Input: "eni-lyft-1", 99 | Response: ec2.DeleteNetworkInterfaceOutput{}, 100 | }, 101 | } 102 | 103 | for i, c := range cases { 104 | defaultClient.ec2Client = &ec2ClientMock{NetworkDeleteResponse: c.Response} 105 | err := defaultClient.deleteInterface(c.Input) 106 | 107 | if err != nil { 108 | t.Fatalf("%d Mock returned an error: %v", i, err) 109 | } 110 | } 111 | } 112 | 113 | func TestWaitUntilInterfaceDetaches(t *testing.T) { 114 | interfaceDetachAttempts = 1 115 | cases := []struct { 116 | Input string 117 | Expected string 118 | Response ec2.DescribeNetworkInterfacesOutput 119 | }{ 120 | { 121 | Input: "eni-lyft-1", 122 | Expected: "", 123 | Response: ec2.DescribeNetworkInterfacesOutput{ 124 | NetworkInterfaces: []*ec2.NetworkInterface{ 125 | { 126 | NetworkInterfaceId: aws.String("eni-lyft-1"), 127 | Status: aws.String("available"), 128 | }, 129 | }, 130 | }, 131 | }, 132 | { 133 | Input: "eni-lyft-2", 134 | Expected: "Interface eni-lyft-2 has not detached yet, use --force to override this check", 135 | Response: ec2.DescribeNetworkInterfacesOutput{ 136 | NetworkInterfaces: []*ec2.NetworkInterface{ 137 | { 138 | NetworkInterfaceId: aws.String("eni-lyft-2"), 139 | Status: aws.String("pending"), 140 | }, 141 | }, 142 | }, 143 | }, 144 | } 145 | 146 | for i, c := range cases { 147 | defaultClient.ec2Client = &ec2ClientMock{NetworkDescribeResponse: c.Response} 148 | err := defaultClient.waitUtilInterfaceDetaches(c.Input) 149 | 150 | if err != nil { 151 | if err.Error() != c.Expected { 152 | t.Fatalf("%d Mock returned an error: %v", i, err) 153 | } 154 | } 155 | } 156 | } 157 | 158 | func TestDescribeNetworkInterface(t *testing.T) { 159 | cases := []struct { 160 | Input string 161 | Expected ec2.NetworkInterface 162 | Response ec2.DescribeNetworkInterfacesOutput 163 | }{ 164 | { 165 | Input: "eni-lyft-1", 166 | Expected: ec2.NetworkInterface{ 167 | NetworkInterfaceId: aws.String("eni-lyft-1"), 168 | }, 169 | Response: ec2.DescribeNetworkInterfacesOutput{ 170 | NetworkInterfaces: []*ec2.NetworkInterface{ 171 | { 172 | NetworkInterfaceId: aws.String("eni-lyft-1"), 173 | }, 174 | }, 175 | }, 176 | }, 177 | { 178 | Input: "eni-lyft-2", 179 | // We actually dont expect anything here, this should throw an error as no 180 | // interfaces were returned 181 | Expected: ec2.NetworkInterface{ 182 | NetworkInterfaceId: aws.String("eni-lyft-"), 183 | }, 184 | Response: ec2.DescribeNetworkInterfacesOutput{ 185 | NetworkInterfaces: []*ec2.NetworkInterface{}, 186 | }, 187 | }, 188 | } 189 | 190 | for i, c := range cases { 191 | defaultClient.ec2Client = &ec2ClientMock{NetworkDescribeResponse: c.Response} 192 | res, err := defaultClient.describeNetworkInterface(c.Input) 193 | 194 | if err != nil { 195 | if err.Error() != "Cannot describe interface, it might not exist" { 196 | t.Fatalf("%d Mock returned an error: %v", i, err) 197 | } 198 | } 199 | 200 | if reflect.DeepEqual(res, c.Expected) { 201 | t.Fatalf("%d DescribeNetworkInterface did not return expected results", i) 202 | } 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /aws/limits.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/ec2" 9 | "github.com/lyft/cni-ipvlan-vpc-k8s/aws/cache" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // ENILimit contains limits for adapter count and addresses 14 | type ENILimit struct { 15 | Adapters int64 16 | IPv4 int64 17 | IPv6 int64 18 | } 19 | 20 | // LimitsClient provides methods for locating limits in AWS 21 | type LimitsClient interface { 22 | ENILimits() (*ENILimit, error) 23 | } 24 | 25 | var defaultLimit = ENILimit{ 26 | Adapters: 4, 27 | IPv4: 15, 28 | IPv6: 15, 29 | } 30 | 31 | // ENILimitsForInstanceType returns the limits for ENI for an instance type 32 | func (c *awsclient) ENILimitsForInstanceType(itype string) (*ENILimit, error) { 33 | client, err := c.newEC2() 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | itypeList := []string{itype} 39 | describeInstanceTypesInput := &ec2.DescribeInstanceTypesInput{ 40 | InstanceTypes: aws.StringSlice(itypeList), 41 | } 42 | 43 | instanceDescribeOutput, err := client.DescribeInstanceTypes(describeInstanceTypesInput) 44 | if err != nil { 45 | return nil, err 46 | } 47 | if len(instanceDescribeOutput.InstanceTypes) == 0 { 48 | return nil, fmt.Errorf("empty answer from DescribeInstanceTypes for %s", itype) 49 | } 50 | 51 | netInfo := instanceDescribeOutput.InstanceTypes[0].NetworkInfo 52 | limit := &ENILimit{ 53 | Adapters: *netInfo.MaximumNetworkInterfaces, 54 | IPv4: *netInfo.Ipv4AddressesPerInterface, 55 | IPv6: *netInfo.Ipv6AddressesPerInterface, 56 | } 57 | return limit, nil 58 | } 59 | 60 | // ENILimits returns the limits based on the system's instance type 61 | func (c *awsclient) ENILimits() (*ENILimit, error) { 62 | id, err := c.getIDDoc() 63 | if err != nil || id == nil { 64 | return &defaultLimit, errors.Wrap(err, "unable get instance identity doc") 65 | } 66 | 67 | // Use the instance type in the cache key in case at some point the cache dir is persisted across reboots 68 | // (instances can be stopped and resized) 69 | key := "eni_limits_for_" + id.InstanceType 70 | limit := &ENILimit{} 71 | if cache.Get(key, limit) == cache.CacheFound { 72 | return limit, nil 73 | } 74 | 75 | limit, err = c.ENILimitsForInstanceType(id.InstanceType) 76 | if err != nil { 77 | return &defaultLimit, errors.Wrap(err, "unable get instance network limits") 78 | } 79 | 80 | cache.Store(key, 24*time.Hour, limit) 81 | return limit, nil 82 | } 83 | -------------------------------------------------------------------------------- /aws/limits_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 9 | "github.com/aws/aws-sdk-go/service/ec2" 10 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 11 | ) 12 | 13 | type ec2InstanceTypesMock struct { 14 | ec2iface.EC2API 15 | InstanceTypesDescribeResp ec2.DescribeInstanceTypesOutput 16 | InstanceTypesDescribeCalls int 17 | } 18 | 19 | func (e *ec2InstanceTypesMock) DescribeInstanceTypes(in *ec2.DescribeInstanceTypesInput) (*ec2.DescribeInstanceTypesOutput, error) { 20 | e.InstanceTypesDescribeCalls++ 21 | return &e.InstanceTypesDescribeResp, nil 22 | } 23 | 24 | func TestLimitsReturn(t *testing.T) { 25 | oldIDDoc := defaultClient.idDoc 26 | defer func() { defaultClient.idDoc = oldIDDoc }() 27 | 28 | cases := []struct { 29 | iType string 30 | Resp ec2.DescribeInstanceTypesOutput 31 | Expected *ENILimit 32 | }{ 33 | { 34 | Expected: &ENILimit{ 35 | Adapters: 4, 36 | IPv4: 15, 37 | IPv6: 15, 38 | }, 39 | iType: "r4.xlarge", 40 | Resp: ec2.DescribeInstanceTypesOutput{ 41 | InstanceTypes: []*ec2.InstanceTypeInfo{ 42 | { 43 | NetworkInfo: &ec2.NetworkInfo{ 44 | Ipv4AddressesPerInterface: aws.Int64(15), 45 | Ipv6AddressesPerInterface: aws.Int64(15), 46 | MaximumNetworkInterfaces: aws.Int64(4), 47 | }, 48 | }, 49 | }, 50 | }, 51 | }, 52 | { 53 | Expected: &ENILimit{ 54 | Adapters: 15, 55 | IPv4: 50, 56 | IPv6: 50, 57 | }, 58 | iType: "c5n.18xlarge", 59 | Resp: ec2.DescribeInstanceTypesOutput{ 60 | InstanceTypes: []*ec2.InstanceTypeInfo{ 61 | { 62 | NetworkInfo: &ec2.NetworkInfo{ 63 | Ipv4AddressesPerInterface: aws.Int64(50), 64 | Ipv6AddressesPerInterface: aws.Int64(50), 65 | MaximumNetworkInterfaces: aws.Int64(15), 66 | }, 67 | }, 68 | }, 69 | }, 70 | }, 71 | } 72 | 73 | for _, c := range cases { 74 | defaultClient.idDoc = &ec2metadata.EC2InstanceIdentityDocument{ 75 | Region: "us-east-1", 76 | AvailabilityZone: "us-east-1a", 77 | InstanceType: c.iType, 78 | } 79 | mock := &ec2InstanceTypesMock{ 80 | InstanceTypesDescribeResp: c.Resp, 81 | } 82 | defaultClient.ec2Client = mock 83 | 84 | res, _ := defaultClient.ENILimits() 85 | if !reflect.DeepEqual(res, c.Expected) { 86 | t.Fatalf("ENILimits do not match. Expected: %v Got: %v", c.Expected, res) 87 | } 88 | 89 | calls := mock.InstanceTypesDescribeCalls 90 | 91 | res, _ = defaultClient.ENILimits() 92 | if mock.InstanceTypesDescribeCalls != calls { 93 | t.Fatalf("Caching logic failed, API call made") 94 | } 95 | if !reflect.DeepEqual(res, c.Expected) { 96 | t.Fatalf("ENILimits from cache do not match. Expected: %v Got: %v", c.Expected, res) 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /aws/metadata.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "sort" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | // Interface describes an interface from the metadata service 13 | type Interface struct { 14 | ID string 15 | Mac string 16 | IfName string 17 | Number int 18 | IPv4s []net.IP 19 | 20 | SubnetID string 21 | SubnetCidr *net.IPNet 22 | 23 | VpcID string 24 | VpcPrimaryCidr *net.IPNet 25 | VpcCidrs []*net.IPNet 26 | SecurityGroupIds []string 27 | } 28 | 29 | // LocalName returns the instance name in string form 30 | func (i Interface) LocalName() string { 31 | return i.IfName 32 | } 33 | 34 | // Interfaces contains a slice of Interface 35 | type Interfaces []Interface 36 | 37 | func (a Interfaces) Len() int { return len(a) } 38 | func (a Interfaces) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 39 | func (a Interfaces) Less(i, j int) bool { return a[i].Number < a[j].Number } 40 | 41 | // MetadataClient provides methods to query the metadata service 42 | type MetadataClient interface { 43 | Available() bool 44 | GetInterfaces() ([]Interface, error) 45 | InstanceType() string 46 | } 47 | 48 | // EC2 generally gives the following data blocks from an interface in meta-data 49 | // device-number 50 | // interface-id 51 | // local-hostname 52 | // local-ipv4s 53 | // mac 54 | // owner-id 55 | // security-group-ids 56 | // security-groups 57 | // subnet-id 58 | // subnet-ipv4-cidr-block 59 | // vpc-id 60 | // vpc-ipv4-cidr-block 61 | // vpc-ipv4-cidr-blocks 62 | // vpc-ipv6-cidr-blocks 63 | 64 | func (c *awsclient) getInterface(mac string) (Interface, error) { 65 | var iface Interface 66 | iface.Mac = mac 67 | 68 | prefix := fmt.Sprintf("network/interfaces/macs/%s/", mac) 69 | get := func(val string) (data string, err error) { 70 | return c.metaData.GetMetadata(fmt.Sprintf("%s/%s", prefix, val)) 71 | } 72 | metadataParser := func(metadataId string, modifer func(*Interface, string) error) error { 73 | metadata, err := get(metadataId) 74 | if err != nil { 75 | log.Printf("Error calling metadata service: %v", err) 76 | return err 77 | } 78 | if metadata != "" { 79 | return modifer(&iface, metadata) 80 | } 81 | return nil 82 | } 83 | 84 | if err := metadataParser("interface-id", func(iface *Interface, value string) error { 85 | iface.ID = value 86 | return nil 87 | }); err != nil { 88 | return iface, err 89 | } 90 | 91 | if err := metadataParser("device-number", func(iface *Interface, value string) error { 92 | num, err := strconv.Atoi(value) 93 | if err != nil { 94 | return err 95 | } 96 | iface.Number = num 97 | return nil 98 | }); err != nil { 99 | return iface, err 100 | } 101 | 102 | if err := metadataParser("local-ipv4s", func(iface *Interface, value string) error { 103 | for _, ipv4 := range strings.Split(value, "\n") { 104 | parsed := net.ParseIP(ipv4) 105 | if parsed != nil { 106 | iface.IPv4s = append(iface.IPv4s, parsed) 107 | } 108 | } 109 | return nil 110 | }); err != nil { 111 | return iface, err 112 | } 113 | 114 | if err := metadataParser("subnet-id", func(iface *Interface, value string) error { 115 | iface.SubnetID = value 116 | return nil 117 | }); err != nil { 118 | return iface, err 119 | } 120 | 121 | if err := metadataParser("subnet-ipv4-cidr-block", func(iface *Interface, value string) error { 122 | var err error 123 | _, iface.SubnetCidr, err = net.ParseCIDR(value) 124 | return err 125 | }); err != nil { 126 | return iface, err 127 | } 128 | 129 | if err := metadataParser("vpc-id", func(iface *Interface, value string) error { 130 | iface.VpcID = value 131 | return nil 132 | }); err != nil { 133 | return iface, err 134 | } 135 | 136 | if err := metadataParser("vpc-ipv4-cidr-block", func(iface *Interface, value string) error { 137 | var err error 138 | _, iface.VpcPrimaryCidr, err = net.ParseCIDR(value) 139 | return err 140 | }); err != nil { 141 | return iface, err 142 | } 143 | 144 | if err := metadataParser("vpc-ipv4-cidr-blocks", func(iface *Interface, value string) error { 145 | cidrList := strings.Split(value, "\n") 146 | if len(cidrList) == 0 { 147 | return fmt.Errorf("No VPC ranges found") 148 | } 149 | for _, vpcCidr := range cidrList { 150 | _, net, err := net.ParseCIDR(vpcCidr) 151 | if err != nil { 152 | return err 153 | } 154 | iface.VpcCidrs = append(iface.VpcCidrs, net) 155 | } 156 | return nil 157 | }); err != nil { 158 | return iface, err 159 | } 160 | 161 | if err := metadataParser("security-group-ids", func(iface *Interface, value string) error { 162 | secGrps := strings.Split(value, "\n") 163 | iface.SecurityGroupIds = secGrps 164 | return nil 165 | }); err != nil { 166 | return iface, err 167 | } 168 | 169 | // Retrieve interface name on host for this MAC address 170 | ifaces, err := net.Interfaces() 171 | if err != nil { 172 | return iface, err 173 | } 174 | for _, i := range ifaces { 175 | if i.HardwareAddr.String() == mac { 176 | iface.IfName = i.Name 177 | break 178 | } 179 | } 180 | // Commented because the AWS metadata server can return MAC addreses from detached interfaces on c5/m5 181 | // A cleaner fix would be to ignore bogus interfaces (but probably not the effort because it should get fixed soon) 182 | //if iface.IfName == "" { 183 | // return iface, fmt.Errorf("Unable to locate interface with mac %s on host", mac) 184 | //} 185 | 186 | return iface, nil 187 | } 188 | 189 | // Available returns the availability status 190 | func (c *awsclient) Available() bool { 191 | return c.metaData.Available() 192 | } 193 | 194 | // GetInterfaces returns a list of configured interfaces 195 | func (c *awsclient) GetInterfaces() ([]Interface, error) { 196 | var interfaces []Interface 197 | 198 | if !c.metaData.Available() { 199 | return nil, fmt.Errorf("EC2 Metadata not available") 200 | } 201 | 202 | macResult, err := c.metaData.GetMetadata("network/interfaces/macs/") 203 | if err != nil { 204 | return nil, err 205 | } 206 | 207 | macs := strings.Split(macResult, "\n") 208 | for _, mac := range macs { 209 | if len(mac) < 1 { 210 | continue 211 | } 212 | mac = mac[0 : len(mac)-1] 213 | iface, err := c.getInterface(mac) 214 | if err != nil { 215 | return nil, err 216 | } 217 | interfaces = append(interfaces, iface) 218 | } 219 | 220 | sort.Sort(Interfaces(interfaces)) 221 | 222 | return interfaces, nil 223 | } 224 | 225 | // InstanceType gets the type of the instance, i.e. "c5.large" 226 | func (c *awsclient) InstanceType() string { 227 | id, err := c.getIDDoc() 228 | if err != nil { 229 | return "unknown" 230 | } 231 | 232 | return id.InstanceType 233 | } 234 | -------------------------------------------------------------------------------- /aws/registry.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "math/rand" 8 | "net" 9 | "os" 10 | "path" 11 | "sync" 12 | "time" 13 | 14 | "github.com/lyft/cni-ipvlan-vpc-k8s/lib" 15 | ) 16 | 17 | const ( 18 | registryDir = "cni-ipvlan-vpc-k8s" 19 | registryFile = "registry.json" 20 | registrySchemaVersion = 1 21 | ) 22 | 23 | func defaultRegistry() registryContents { 24 | return registryContents{ 25 | SchemaVersion: registrySchemaVersion, 26 | IPs: map[string]*registryIP{}, 27 | } 28 | } 29 | 30 | type registryIP struct { 31 | ReleasedOn lib.JSONTime `json:"released_on"` 32 | } 33 | 34 | type registryContents struct { 35 | SchemaVersion int `json:"schema_version"` 36 | IPs map[string]*registryIP `json:"ips"` 37 | } 38 | 39 | // Registry defines a re-usable IP registry which tracks IPs that are 40 | // free in the system and when they were last released back to the pool. 41 | type Registry struct { 42 | path string 43 | lock sync.Mutex 44 | } 45 | 46 | // registryPath gives a default location for the registry 47 | // which varies based on invoking user ID 48 | func registryPath() string { 49 | uid := os.Getuid() 50 | if uid != 0 { 51 | // Non-root users of the registry 52 | return path.Join("/run/user", fmt.Sprintf("%d", uid), registryDir) 53 | } 54 | 55 | return path.Join("/run", registryDir) 56 | } 57 | 58 | func (r *Registry) ensurePath() (string, error) { 59 | if len(r.path) == 0 { 60 | r.path = registryPath() 61 | } 62 | rpath := r.path 63 | info, err := os.Stat(rpath) 64 | if err != nil || !info.IsDir() { 65 | err = os.MkdirAll(rpath, os.ModeDir|0700) 66 | if err != nil { 67 | return "", err 68 | } 69 | } 70 | rpath = path.Join(rpath, registryFile) 71 | return rpath, nil 72 | } 73 | 74 | func (r *Registry) load() (*registryContents, error) { 75 | // Load the pre-versioned schema 76 | contents := defaultRegistry() 77 | rpath, err := r.ensurePath() 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | file, err := os.Open(rpath) 83 | if os.IsNotExist(err) { 84 | // Return an empty registry, prefilled with IPs 85 | // already existing on all interfaces and timestamped 86 | // at the golang epoch 87 | free, err := FindFreeIPsAtIndex(0, false) 88 | if err == nil || len(free) > 0 { 89 | for _, freeAlloc := range free { 90 | contents.IPs[freeAlloc.IP.String()] = ®istryIP{ 91 | ReleasedOn: lib.JSONTime{Time: time.Time{}}, 92 | } 93 | } 94 | err = r.save(&contents) 95 | return &contents, err 96 | } 97 | return &contents, nil 98 | } else if err != nil { 99 | return nil, err 100 | } 101 | 102 | defer file.Close() 103 | 104 | decoder := json.NewDecoder(file) 105 | if decoder == nil { 106 | return nil, fmt.Errorf("invalid decoder") 107 | } 108 | 109 | err = decoder.Decode(&contents) 110 | if err != nil { 111 | log.Printf("invalid registry format, returning empty registry %v", err) 112 | contents = defaultRegistry() 113 | } 114 | 115 | // Reset the registry if the version is not what we can deal with. 116 | // Add more states here, or just let the registry be blown away 117 | // on invalid loads 118 | if contents.SchemaVersion != registrySchemaVersion { 119 | contents = defaultRegistry() 120 | } 121 | if contents.IPs == nil { 122 | contents = defaultRegistry() 123 | } 124 | return &contents, nil 125 | } 126 | 127 | func (r *Registry) save(rc *registryContents) error { 128 | rpath, err := r.ensurePath() 129 | if err != nil { 130 | return err 131 | } 132 | file, err := os.OpenFile(rpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) 133 | if err != nil { 134 | return err 135 | } 136 | defer file.Close() 137 | 138 | encoder := json.NewEncoder(file) 139 | if encoder == nil { 140 | return fmt.Errorf("could not make a new encoder") 141 | } 142 | rc.SchemaVersion = registrySchemaVersion 143 | err = encoder.Encode(rc) 144 | return err 145 | } 146 | 147 | // TrackIPAtEpoch sets the IP recorded time as the epoch (0 time) 148 | // so it appears as immediately free and avoids re-allocation 149 | func (r *Registry) TrackIPAtEpoch(ip net.IP) error { 150 | r.lock.Lock() 151 | defer r.lock.Unlock() 152 | 153 | contents, err := r.load() 154 | if err != nil { 155 | return err 156 | } 157 | 158 | contents.IPs[ip.String()] = ®istryIP{ 159 | ReleasedOn: lib.JSONTime{Time: time.Time{}}, 160 | } 161 | return r.save(contents) 162 | } 163 | 164 | // TrackIP records an IP in the free registry with the current system 165 | // time as the current freed-time. If an IP is freed again, the time 166 | // will be updated to the new current time. 167 | func (r *Registry) TrackIP(ip net.IP) error { 168 | r.lock.Lock() 169 | defer r.lock.Unlock() 170 | 171 | contents, err := r.load() 172 | if err != nil { 173 | return err 174 | } 175 | 176 | contents.IPs[ip.String()] = ®istryIP{ 177 | ReleasedOn: lib.JSONTime{Time: time.Now()}, 178 | } 179 | return r.save(contents) 180 | } 181 | 182 | // ForgetIP removes an IP from the registry 183 | func (r *Registry) ForgetIP(ip net.IP) error { 184 | r.lock.Lock() 185 | defer r.lock.Unlock() 186 | 187 | contents, err := r.load() 188 | if err != nil { 189 | return err 190 | } 191 | 192 | delete(contents.IPs, ip.String()) 193 | 194 | return r.save(contents) 195 | } 196 | 197 | // HasIP checks if an IP is in an registry 198 | func (r *Registry) HasIP(ip net.IP) (bool, error) { 199 | r.lock.Lock() 200 | defer r.lock.Unlock() 201 | 202 | contents, err := r.load() 203 | if err != nil { 204 | return false, err 205 | } 206 | 207 | _, ok := contents.IPs[ip.String()] 208 | return ok, nil 209 | } 210 | 211 | // TrackedBefore returns a list of all IPs last recorded time _before_ 212 | // the time passed to this function. You probably want to call this 213 | // with time.Now().Add(-duration). 214 | func (r *Registry) TrackedBefore(t time.Time) ([]net.IP, error) { 215 | r.lock.Lock() 216 | defer r.lock.Unlock() 217 | 218 | contents, err := r.load() 219 | if err != nil { 220 | return nil, err 221 | } 222 | 223 | returned := []net.IP{} 224 | for ipString, entry := range contents.IPs { 225 | if entry.ReleasedOn.Before(t) { 226 | ip := net.ParseIP(ipString) 227 | if ip == nil { 228 | continue 229 | } 230 | returned = append(returned, ip) 231 | } 232 | } 233 | return returned, nil 234 | } 235 | 236 | // Clear clears the registry unconditionally 237 | func (r *Registry) Clear() error { 238 | r.lock.Lock() 239 | defer r.lock.Unlock() 240 | 241 | rpath, err := r.ensurePath() 242 | if err != nil { 243 | return err 244 | } 245 | 246 | err = os.Remove(rpath) 247 | if os.IsNotExist(err) { 248 | return nil 249 | } 250 | 251 | return err 252 | } 253 | 254 | // List returns a list of all tracked IPs 255 | func (r *Registry) List() (ret []net.IP, err error) { 256 | r.lock.Lock() 257 | defer r.lock.Unlock() 258 | 259 | contents, err := r.load() 260 | if err != nil { 261 | return nil, err 262 | } 263 | for stringIP := range contents.IPs { 264 | ret = append(ret, net.ParseIP(stringIP)) 265 | } 266 | return 267 | } 268 | 269 | // Jitter takes a duration and adjusts it forward by a up to `pct` percent 270 | // uniformly. 271 | func Jitter(d time.Duration, pct float64) time.Duration { 272 | jitter := rand.Int63n(int64(float64(d) * pct)) 273 | d += time.Duration(jitter) 274 | return d 275 | } 276 | -------------------------------------------------------------------------------- /aws/registry_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | const ( 10 | IP1 = "127.0.0.1" 11 | IP2 = "127.0.0.2" 12 | IP3 = "127.0.0.3" 13 | ) 14 | 15 | func TestRegistry_TrackIp(t *testing.T) { 16 | r := &Registry{} 17 | 18 | err := r.TrackIP(net.ParseIP(IP1)) 19 | if err != nil { 20 | t.Fatalf("Failed to track IP %v", err) 21 | } 22 | 23 | if ok, err := r.HasIP(net.ParseIP(IP1)); !ok || err != nil { 24 | t.Fatalf("Did not remember IP %v %v %v", IP1, ok, err) 25 | } 26 | } 27 | 28 | func TestRegistry_Clear(t *testing.T) { 29 | r := &Registry{} 30 | 31 | err := r.Clear() 32 | if err != nil { 33 | t.Fatalf("clear failed %v", err) 34 | } 35 | 36 | _ = r.TrackIP(net.ParseIP(IP3)) 37 | if ok, err := r.HasIP(net.ParseIP(IP3)); !ok || err != nil { 38 | t.Fatalf("Did not remember IP %v %v %v", IP3, ok, err) 39 | } 40 | 41 | err = r.Clear() 42 | if err != nil { 43 | t.Fatalf("clear failed %v", err) 44 | } 45 | 46 | if ok, err := r.HasIP(net.ParseIP(IP3)); ok || err != nil { 47 | t.Fatalf("Did not forget IP %v %v %v", IP3, ok, err) 48 | } 49 | } 50 | 51 | func TestRegistry_ForgetIP(t *testing.T) { 52 | r := &Registry{} 53 | 54 | err := r.Clear() 55 | if err != nil { 56 | t.Fatalf("clear failed %v", err) 57 | } 58 | 59 | _ = r.TrackIP(net.ParseIP(IP2)) 60 | if ok, err := r.HasIP(net.ParseIP(IP2)); !ok || err != nil { 61 | t.Fatalf("Did not remember IP %v %v %v", IP3, ok, err) 62 | } 63 | 64 | err = r.ForgetIP(net.ParseIP(IP2)) 65 | if err != nil { 66 | t.Fatalf("forget failed %v", err) 67 | } 68 | 69 | if ok, err := r.HasIP(net.ParseIP(IP2)); ok || err != nil { 70 | t.Fatalf("Did not forget IP %v %v %v", IP2, ok, err) 71 | } 72 | 73 | // Forget an IP never registered 74 | err = r.ForgetIP(net.ParseIP(IP1)) 75 | if err != nil { 76 | t.Fatalf("forgetting an IP not tracked should not be an error") 77 | } 78 | 79 | } 80 | 81 | func TestRegistry_TrackedBefore(t *testing.T) { 82 | r := &Registry{} 83 | 84 | err := r.Clear() 85 | if err != nil { 86 | t.Fatalf("clear failed %v", err) 87 | } 88 | 89 | _ = r.TrackIP(net.ParseIP(IP1)) 90 | now := time.Now() 91 | 92 | before, err := r.TrackedBefore(now.Add(100 * time.Hour)) 93 | if err != nil { 94 | t.Fatalf("error tracked before %v", err) 95 | } 96 | 97 | if len(before) != 1 { 98 | t.Fatalf("Invalid number of entries, got %v", before) 99 | } 100 | 101 | after, err := r.TrackedBefore(now.Add(-100 * time.Hour)) 102 | if err != nil { 103 | t.Fatalf("error tracked before %v", err) 104 | } 105 | 106 | if len(after) != 0 { 107 | t.Fatalf("Should return no IPs before the future, got %v", after) 108 | } 109 | } 110 | 111 | func TestJitter(t *testing.T) { 112 | d1 := 1 * time.Second 113 | d1p := Jitter(d1, 0.10) 114 | if d1 >= d1p { 115 | t.Fatalf("Jitter did not move forward: %v %v", d1, d1p) 116 | } 117 | 118 | if d1p > 1101*time.Millisecond { 119 | t.Fatalf("Jitter moved more than 10pct forward %v", d1p) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /aws/subnets.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 7 | "github.com/aws/aws-sdk-go/service/ec2" 8 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 9 | "github.com/lyft/cni-ipvlan-vpc-k8s/aws/cache" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // Subnet contains attributes of a subnet 14 | type Subnet struct { 15 | ID string 16 | Cidr string 17 | IsDefault bool 18 | AvailableAddressCount int 19 | Name string 20 | Tags map[string]string 21 | } 22 | 23 | // SubnetsByAvailableAddressCount contains a list of subnet 24 | type SubnetsByAvailableAddressCount []Subnet 25 | 26 | func (a SubnetsByAvailableAddressCount) Len() int { return len(a) } 27 | func (a SubnetsByAvailableAddressCount) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 28 | func (a SubnetsByAvailableAddressCount) Less(i, j int) bool { 29 | return a[i].AvailableAddressCount > a[j].AvailableAddressCount 30 | } 31 | 32 | // SubnetsClient provides information about VPC subnets 33 | type SubnetsClient interface { 34 | GetSubnetsForInstance() ([]Subnet, error) 35 | } 36 | 37 | type subnetsCacheClient struct { 38 | subnets *subnetsClient 39 | expiration time.Duration 40 | } 41 | 42 | func (s *subnetsCacheClient) GetSubnetsForInstance() (subnets []Subnet, err error) { 43 | state := cache.Get("subnets_for_instance", &subnets) 44 | if state == cache.CacheFound { 45 | return 46 | } 47 | subnets, err = s.subnets.GetSubnetsForInstance() 48 | if err == nil { 49 | cache.Store("subnets_for_instance", s.expiration, &subnets) 50 | } 51 | return 52 | } 53 | 54 | type subnetsClient struct { 55 | aws awsSubnetClient 56 | } 57 | 58 | type awsSubnetClient interface { 59 | getIDDoc() (*ec2metadata.EC2InstanceIdentityDocument, error) 60 | newEC2() (ec2iface.EC2API, error) 61 | GetInterfaces() ([]Interface, error) 62 | } 63 | 64 | // GetSubnetsForInstance returns a list of subnets for the running instance 65 | func (c *subnetsClient) GetSubnetsForInstance() ([]Subnet, error) { 66 | var subnets []Subnet 67 | 68 | id, err := c.aws.getIDDoc() 69 | if err != nil { 70 | return nil, err 71 | } 72 | az := id.AvailabilityZone 73 | 74 | ec2Client, err := c.aws.newEC2() 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | // getting all interfaces attached to this specific machine so we can find out what is our vpc-id 80 | // interfaces[0] is going to be our eth0, interfaces slice gets sorted by number before returning to the caller 81 | interfaces, err := c.aws.GetInterfaces() 82 | if err != nil { 83 | return nil, errors.Wrap(err, "failed to get interfaces associated with this EC2 machine") 84 | } 85 | 86 | input := &ec2.DescribeSubnetsInput{} 87 | input.Filters = []*ec2.Filter{ 88 | newEc2Filter("vpc-id", interfaces[0].VpcID), 89 | newEc2Filter("availabilityZone", az), 90 | } 91 | 92 | result, err := ec2Client.DescribeSubnets(input) 93 | 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | for _, awsSub := range result.Subnets { 99 | subnet := Subnet{ 100 | ID: *awsSub.SubnetId, 101 | Cidr: *awsSub.CidrBlock, 102 | IsDefault: *awsSub.DefaultForAz, 103 | AvailableAddressCount: int(*awsSub.AvailableIpAddressCount), 104 | Tags: map[string]string{}, 105 | } 106 | // Set all the tags on the result 107 | for _, tag := range awsSub.Tags { 108 | if *tag.Key == "Name" { 109 | subnet.Name = *tag.Value 110 | } else { 111 | subnet.Tags[*tag.Key] = *tag.Value 112 | } 113 | } 114 | subnets = append(subnets, subnet) 115 | } 116 | 117 | return subnets, nil 118 | } 119 | -------------------------------------------------------------------------------- /aws/subnets_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 8 | "github.com/aws/aws-sdk-go/service/ec2" 9 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 10 | ) 11 | 12 | type awsSubnetClientMock struct { 13 | EC2Client ec2iface.EC2API 14 | InterfaceResponse []Interface 15 | IDDocument *ec2metadata.EC2InstanceIdentityDocument 16 | } 17 | 18 | func (m awsSubnetClientMock) getIDDoc() (*ec2metadata.EC2InstanceIdentityDocument, error) { 19 | return m.IDDocument, nil 20 | } 21 | 22 | func (m awsSubnetClientMock) newEC2() (ec2iface.EC2API, error) { 23 | return m.EC2Client, nil 24 | } 25 | 26 | func (m awsSubnetClientMock) GetInterfaces() ([]Interface, error) { 27 | return m.InterfaceResponse, nil 28 | } 29 | 30 | type ec2SubnetsMock struct { 31 | ec2iface.EC2API 32 | Resp ec2.DescribeSubnetsOutput 33 | } 34 | 35 | func (e *ec2SubnetsMock) DescribeSubnets(in *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) { 36 | return &e.Resp, nil 37 | } 38 | 39 | func TestGetSubnetsForInstance(t *testing.T) { 40 | cases := []struct { 41 | Resp ec2.DescribeSubnetsOutput 42 | Expected int 43 | }{ 44 | { 45 | Expected: 1, 46 | Resp: ec2.DescribeSubnetsOutput{ 47 | Subnets: []*ec2.Subnet{ 48 | { 49 | AvailabilityZone: aws.String("us-east-1a"), 50 | CidrBlock: aws.String("192.168.0.0/24"), 51 | DefaultForAz: aws.Bool(false), 52 | SubnetId: aws.String("subnet-1234"), 53 | AvailableIpAddressCount: aws.Int64(12), 54 | Tags: []*ec2.Tag{ 55 | { 56 | Key: aws.String("Name"), 57 | Value: aws.String("subnet 1"), 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, 63 | }, 64 | } 65 | 66 | for i, c := range cases { 67 | defaultClient.subnets = &subnetsClient{ 68 | aws: &awsSubnetClientMock{ 69 | EC2Client: &ec2SubnetsMock{Resp: c.Resp}, 70 | IDDocument: &ec2metadata.EC2InstanceIdentityDocument{ 71 | Region: "us-east-1", 72 | AvailabilityZone: "us-east-1a", 73 | }, 74 | InterfaceResponse: []Interface{ 75 | { 76 | VpcID: "vpc-id", 77 | }, 78 | }, 79 | }, 80 | } 81 | 82 | res, err := defaultClient.GetSubnetsForInstance() 83 | if err != nil { 84 | t.Fatalf("%d Mock returned an error - is it mocked? %v", i, err) 85 | } 86 | 87 | if len(res) != c.Expected { 88 | t.Fatalf("%d Subnets not all returned", i) 89 | } 90 | 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /aws/util.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/service/ec2" 6 | ) 7 | 8 | func newEc2Filter(name string, values ...string) *ec2.Filter { 9 | filter := &ec2.Filter{ 10 | Name: aws.String(name), 11 | } 12 | for _, value := range values { 13 | filter.Values = append(filter.Values, aws.String(value)) 14 | } 15 | return filter 16 | } 17 | -------------------------------------------------------------------------------- /aws/util_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/ec2" 9 | ) 10 | 11 | func TestNewEc2Filter(t *testing.T) { 12 | type FilterInput struct { 13 | Name string 14 | Values []string 15 | } 16 | 17 | cases := []struct { 18 | Input FilterInput 19 | Expect ec2.Filter 20 | }{ 21 | { 22 | Input: FilterInput{ 23 | Name: "filter1", 24 | Values: []string{"value1"}, 25 | }, 26 | Expect: ec2.Filter{ 27 | Name: aws.String("filter1"), 28 | Values: aws.StringSlice([]string{"value1"}), 29 | }, 30 | }, 31 | { 32 | Input: FilterInput{ 33 | Name: "filter2", 34 | Values: []string{"value1", "value3", "value3"}, 35 | }, 36 | Expect: ec2.Filter{ 37 | Name: aws.String("filter2"), 38 | Values: aws.StringSlice([]string{"value1", "value3", "value3"}), 39 | }, 40 | }, 41 | } 42 | 43 | for i, c := range cases { 44 | filterOutput := newEc2Filter(c.Input.Name, c.Input.Values...) 45 | 46 | if reflect.DeepEqual(filterOutput, c.Expect) { 47 | t.Fatalf("%d newEc2Filter did not return expected results", i) 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /aws/vpc.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/ec2" 10 | 11 | "github.com/lyft/cni-ipvlan-vpc-k8s/aws/cache" 12 | ) 13 | 14 | // VPCClient provides a view into a VPC 15 | type VPCClient interface { 16 | DescribeVPCCIDRs(vpcID string) ([]*net.IPNet, error) 17 | DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error) 18 | } 19 | 20 | type vpcCacheClient struct { 21 | vpc *vpcclient 22 | expiration time.Duration 23 | } 24 | 25 | func (v *vpcCacheClient) DescribeVPCCIDRs(vpcID string) (cidrs []*net.IPNet, err error) { 26 | key := fmt.Sprintf("vpc-cidr-%v", vpcID) 27 | state := cache.Get(key, &cidrs) 28 | if state == cache.CacheFound { 29 | return 30 | } 31 | cidrs, err = v.vpc.DescribeVPCCIDRs(vpcID) 32 | if err != nil { 33 | return nil, err 34 | } 35 | cache.Store(key, v.expiration, &cidrs) 36 | return 37 | } 38 | 39 | func (v *vpcCacheClient) DescribeVPCPeerCIDRs(vpcID string) (cidrs []*net.IPNet, err error) { 40 | key := fmt.Sprintf("vpc-peers-%v", vpcID) 41 | state := cache.Get(key, &cidrs) 42 | if state == cache.CacheFound { 43 | return 44 | } 45 | cidrs, err = v.vpc.DescribeVPCPeerCIDRs(vpcID) 46 | if err != nil { 47 | return nil, err 48 | } 49 | cache.Store(key, v.expiration, &cidrs) 50 | return 51 | 52 | } 53 | 54 | type vpcclient struct { 55 | aws *awsclient 56 | } 57 | 58 | // DescribeVPCCIDRs returns a list of all CIDRS associated with a VPC 59 | func (v *vpcclient) DescribeVPCCIDRs(vpcID string) ([]*net.IPNet, error) { 60 | req := &ec2.DescribeVpcsInput{ 61 | VpcIds: []*string{aws.String(vpcID)}, 62 | } 63 | ec2, err := v.aws.newEC2() 64 | if err != nil { 65 | return nil, err 66 | } 67 | res, err := ec2.DescribeVpcs(req) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | var cidrs []*net.IPNet 73 | 74 | for _, vpc := range res.Vpcs { 75 | for _, cblock := range vpc.CidrBlockAssociationSet { 76 | // Avoid adding non associated CIDRs to routes 77 | if *cblock.CidrBlockState.State != "associated" { 78 | continue 79 | } 80 | _, parsed, err := net.ParseCIDR(*cblock.CidrBlock) 81 | if err != nil { 82 | return nil, err 83 | } 84 | cidrs = append(cidrs, parsed) 85 | } 86 | } 87 | return cidrs, nil 88 | } 89 | 90 | // DescribeVPCPeerCIDRs returns a list of CIDRs for all peered VPCs to the given VPC 91 | func (v *vpcclient) DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error) { 92 | ec2c, err := v.aws.newEC2() 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | req := &ec2.DescribeVpcPeeringConnectionsInput{} 98 | 99 | res, err := ec2c.DescribeVpcPeeringConnections(req) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | // In certain peering situations, a CIDR may be duplicated 105 | // and visible to the API, even if the CIDR is not active in 106 | // one of the peered VPCs. We store all of the CIDRs in a map 107 | // to de-duplicate them. 108 | cidrs := make(map[string]bool) 109 | 110 | for _, peering := range res.VpcPeeringConnections { 111 | var peer *ec2.VpcPeeringConnectionVpcInfo 112 | 113 | if vpcID == *peering.AccepterVpcInfo.VpcId { 114 | peer = peering.RequesterVpcInfo 115 | } else if vpcID == *peering.RequesterVpcInfo.VpcId { 116 | peer = peering.AccepterVpcInfo 117 | } else { 118 | continue 119 | } 120 | 121 | for _, cidrBlock := range peer.CidrBlockSet { 122 | _, _, err := net.ParseCIDR(*cidrBlock.CidrBlock) 123 | if err == nil { 124 | cidrs[*cidrBlock.CidrBlock] = true 125 | } 126 | } 127 | } 128 | 129 | var returnCidrs []*net.IPNet 130 | for cidrString := range cidrs { 131 | _, cidr, err := net.ParseCIDR(cidrString) 132 | if err == nil && cidr != nil { 133 | returnCidrs = append(returnCidrs, cidr) 134 | } 135 | } 136 | return returnCidrs, nil 137 | } 138 | -------------------------------------------------------------------------------- /cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "strings" 8 | "text/tabwriter" 9 | "time" 10 | 11 | "github.com/urfave/cli" 12 | 13 | "github.com/lyft/cni-ipvlan-vpc-k8s/aws" 14 | "github.com/lyft/cni-ipvlan-vpc-k8s/lib" 15 | "github.com/lyft/cni-ipvlan-vpc-k8s/nl" 16 | ) 17 | 18 | var version string 19 | 20 | // Build a filter from input 21 | func filterBuild(input string) (map[string]string, error) { 22 | if input == "" { 23 | return nil, nil 24 | } 25 | 26 | ret := make(map[string]string) 27 | tuples := strings.Split(input, ",") 28 | for _, t := range tuples { 29 | kv := strings.Split(t, "=") 30 | if len(kv) != 2 { 31 | return nil, fmt.Errorf("Invalid filter specified %v", t) 32 | } 33 | if len(kv[0]) <= 0 || len(kv[1]) <= 0 { 34 | return nil, fmt.Errorf("Zero length filter specified: %v", t) 35 | } 36 | 37 | ret[kv[0]] = kv[1] 38 | } 39 | 40 | return ret, nil 41 | } 42 | 43 | func actionNewInterface(c *cli.Context) error { 44 | return lib.LockfileRun(func() error { 45 | filtersRaw := c.String("subnet_filter") 46 | filters, err := filterBuild(filtersRaw) 47 | if err != nil { 48 | fmt.Printf("Invalid filter specification %v", err) 49 | return err 50 | } 51 | ipBatchSize := c.Int64("ip_batch_size") 52 | 53 | secGrps := c.Args() 54 | 55 | if len(secGrps) <= 0 { 56 | fmt.Println("please specify security groups") 57 | return fmt.Errorf("need security groups") 58 | } 59 | newIf, err := aws.DefaultClient.NewInterface(secGrps, filters, ipBatchSize) 60 | if err != nil { 61 | fmt.Println(err) 62 | return err 63 | } 64 | fmt.Println(newIf) 65 | return nil 66 | 67 | }) 68 | } 69 | 70 | func actionBugs(c *cli.Context) error { 71 | return lib.LockfileRun(func() error { 72 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 73 | fmt.Fprintln(w, "bug\tafflicted\t") 74 | for _, bug := range aws.ListBugs(aws.DefaultClient) { 75 | fmt.Fprintf(w, "%s\t%v\t\n", bug.Name, bug.HasBug()) 76 | } 77 | w.Flush() 78 | return nil 79 | }) 80 | } 81 | 82 | func actionRemoveInterface(c *cli.Context) error { 83 | return lib.LockfileRun(func() error { 84 | interfaces := c.Args() 85 | 86 | if len(interfaces) <= 0 { 87 | fmt.Println("please specify an interface") 88 | return fmt.Errorf("Insufficient Arguments") 89 | } 90 | 91 | if err := aws.DefaultClient.RemoveInterface(interfaces); err != nil { 92 | fmt.Println(err) 93 | return err 94 | } 95 | 96 | return nil 97 | }) 98 | } 99 | 100 | func actionDeallocate(c *cli.Context) error { 101 | return lib.LockfileRun(func() error { 102 | releaseIps := c.Args() 103 | for _, toRelease := range releaseIps { 104 | 105 | if len(toRelease) < 6 { 106 | fmt.Println("please specify an IP") 107 | return fmt.Errorf("Invalid IP") 108 | } 109 | 110 | ip := net.ParseIP(toRelease) 111 | if ip == nil { 112 | fmt.Println("please specify a valid IP") 113 | return fmt.Errorf("IP parse error") 114 | } 115 | 116 | err := aws.DefaultClient.DeallocateIP(&ip) 117 | if err != nil { 118 | fmt.Printf("deallocation failed: %v\n", err) 119 | return err 120 | } 121 | } 122 | return nil 123 | }) 124 | } 125 | 126 | func actionAllocate(c *cli.Context) error { 127 | return lib.LockfileRun(func() error { 128 | index := c.Int("index") 129 | ipBatchSize := c.Int64("ip_batch_size") 130 | res, err := aws.DefaultClient.AllocateIPsFirstAvailableAtIndex(index, ipBatchSize) 131 | for _, alloc := range res { 132 | fmt.Printf("allocated %v on %v\n", alloc.IP, alloc.Interface.LocalName()) 133 | } 134 | 135 | return err 136 | }) 137 | } 138 | 139 | func actionFreeIps(c *cli.Context) error { 140 | ips, err := aws.FindFreeIPsAtIndex(0, false) 141 | if err != nil { 142 | fmt.Println(err) 143 | } 144 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 145 | fmt.Fprintln(w, "adapter\tip\t") 146 | for _, ip := range ips { 147 | fmt.Fprintf(w, "%v\t%v\t\n", 148 | ip.Interface.LocalName(), 149 | ip.IP) 150 | } 151 | w.Flush() 152 | return err 153 | } 154 | 155 | func actionLimits(c *cli.Context) error { 156 | limit, err := aws.DefaultClient.ENILimits() 157 | if err != nil { 158 | return err 159 | } 160 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 161 | fmt.Fprintln(w, "adapters\tipv4\tipv6\t") 162 | fmt.Fprintf(w, "%v\t%v\t%v\t\n", limit.Adapters, 163 | limit.IPv4, 164 | limit.IPv6) 165 | w.Flush() 166 | return nil 167 | } 168 | 169 | func actionMaxPods(c *cli.Context) error { 170 | limit, err := aws.DefaultClient.ENILimits() 171 | retErr := c.Bool("return-error") 172 | if err != nil { 173 | if retErr { 174 | return err 175 | } 176 | fmt.Fprintf(os.Stderr, "unable to determine ENI limit due to '%v', defaulting max pods to 110", err) 177 | fmt.Printf("%d\n", 110) 178 | return nil 179 | } 180 | specifiedMax := int64(c.Int("max")) 181 | max := (limit.Adapters - 1) * limit.IPv4 182 | if specifiedMax > 0 && specifiedMax < max { 183 | // Limit the maximum to the CLI maximum 184 | max = specifiedMax 185 | } 186 | fmt.Printf("%d\n", max) 187 | return nil 188 | } 189 | 190 | func actionAddr(c *cli.Context) error { 191 | ips, err := nl.GetIPs() 192 | if err != nil { 193 | fmt.Println(err) 194 | return err 195 | } 196 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 197 | fmt.Fprintln(w, "iface\tip\t") 198 | for _, ip := range ips { 199 | fmt.Fprintf(w, "%v\t%v\t\n", 200 | ip.Label, 201 | ip.IPNet.IP) 202 | } 203 | w.Flush() 204 | 205 | return nil 206 | } 207 | 208 | func actionEniIf(c *cli.Context) error { 209 | interfaces, err := aws.DefaultClient.GetInterfaces() 210 | if err != nil { 211 | fmt.Println(err) 212 | return err 213 | } 214 | 215 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 216 | fmt.Fprintln(w, "iface\tmac\tid\tsubnet\tsubnet_cidr\tsecgrps\tvpc\tips\t") 217 | for _, iface := range interfaces { 218 | fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t\n", iface.LocalName(), 219 | iface.Mac, 220 | iface.ID, 221 | iface.SubnetID, 222 | iface.SubnetCidr, 223 | iface.SecurityGroupIds, 224 | iface.VpcID, 225 | iface.IPv4s) 226 | 227 | } 228 | 229 | w.Flush() 230 | return nil 231 | } 232 | 233 | func actionVpcCidr(c *cli.Context) error { 234 | interfaces, err := aws.DefaultClient.GetInterfaces() 235 | if err != nil { 236 | fmt.Println(err) 237 | return err 238 | } 239 | 240 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 241 | fmt.Fprintln(w, "iface\tmetadata cidr\taws api cidr\t") 242 | for _, iface := range interfaces { 243 | apiCidrs, _ := aws.DefaultClient.DescribeVPCCIDRs(iface.VpcID) 244 | 245 | fmt.Fprintf(w, "%s\t%v\t%v\t\n", 246 | iface.LocalName(), 247 | iface.VpcCidrs, 248 | apiCidrs) 249 | } 250 | w.Flush() 251 | return nil 252 | } 253 | 254 | func actionVpcPeerCidr(c *cli.Context) error { 255 | interfaces, err := aws.DefaultClient.GetInterfaces() 256 | if err != nil { 257 | fmt.Println(err) 258 | return err 259 | } 260 | 261 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 262 | fmt.Fprintln(w, "iface\tpeer_dcidr\t") 263 | for _, iface := range interfaces { 264 | apiCidrs, _ := aws.DefaultClient.DescribeVPCPeerCIDRs(iface.VpcID) 265 | 266 | fmt.Fprintf(w, "%s\t%v\t\n", 267 | iface.LocalName(), 268 | apiCidrs) 269 | } 270 | w.Flush() 271 | return nil 272 | } 273 | 274 | func actionSubnets(c *cli.Context) error { 275 | subnets, err := aws.DefaultClient.GetSubnetsForInstance() 276 | if err != nil { 277 | fmt.Println(err) 278 | return err 279 | } 280 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 281 | fmt.Fprintln(w, "id\tcidr\tdefault\taddresses_available\ttags\t") 282 | for _, subnet := range subnets { 283 | fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t\n", 284 | subnet.ID, 285 | subnet.Cidr, 286 | subnet.IsDefault, 287 | subnet.AvailableAddressCount, 288 | subnet.Tags) 289 | } 290 | 291 | w.Flush() 292 | 293 | return nil 294 | } 295 | 296 | func actionRegistryList(c *cli.Context) error { 297 | return lib.LockfileRun(func() error { 298 | 299 | reg := &aws.Registry{} 300 | ips, err := reg.List() 301 | if err != nil { 302 | return err 303 | } 304 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 305 | fmt.Fprintln(w, "ip\t") 306 | for _, ip := range ips { 307 | fmt.Fprintf(w, "%v\t\n", 308 | ip) 309 | } 310 | w.Flush() 311 | return nil 312 | }) 313 | } 314 | 315 | func actionRegistryGc(c *cli.Context) error { 316 | return lib.LockfileRun(func() error { 317 | 318 | maxReap := c.Int("max-reap") 319 | 320 | reg := &aws.Registry{} 321 | freeAfter := c.Duration("free-after") 322 | if freeAfter <= 0*time.Second { 323 | fmt.Fprintf(os.Stderr, 324 | "Invalid duration specified. free-after must be > 0 seconds. Got %v. Please specify with --free-after=[time]\n", freeAfter) 325 | return fmt.Errorf("invalid duration") 326 | } 327 | 328 | // Insert free-after jitter of 15% of the period 329 | freeAfter = aws.Jitter(freeAfter, 0.15) 330 | 331 | // Invert free-after 332 | freeAfter *= -1 333 | 334 | ips, err := reg.TrackedBefore(time.Now().Add(freeAfter)) 335 | if err != nil { 336 | fmt.Fprintln(os.Stderr, err) 337 | return err 338 | } 339 | 340 | // grab a list of in-use IPs to sanity check 341 | assigned, err := nl.GetIPs() 342 | if err != nil { 343 | return err 344 | } 345 | 346 | OUTER: 347 | for i, ip := range ips { 348 | // forget IPs that are actually in use and skip over 349 | for _, assignedIP := range assigned { 350 | if assignedIP.IPNet.IP.Equal(ip) { 351 | err = reg.ForgetIP(ip) 352 | if err != nil { 353 | fmt.Fprintf(os.Stderr, "failed to forget %v due to %v", ip, err) 354 | } 355 | continue OUTER 356 | } 357 | } 358 | err := aws.DefaultClient.DeallocateIP(&ips[i]) 359 | if err == nil { 360 | err = reg.ForgetIP(ip) 361 | if err != nil { 362 | fmt.Fprintf(os.Stderr, "failed to forget %v due to %v", ip, err) 363 | } 364 | maxReap-- 365 | } else { 366 | fmt.Fprintf(os.Stderr, "Can't deallocate %v due to %v", ip, err) 367 | } 368 | // max-reap specified as negative number will never reach 0 and reap all unused IPs 369 | if maxReap == 0 { 370 | return nil 371 | } 372 | } 373 | 374 | return nil 375 | }) 376 | } 377 | 378 | func main() { 379 | if !aws.DefaultClient.Available() { 380 | fmt.Fprintln(os.Stderr, "This command must be run from a running ec2 instance") 381 | os.Exit(1) 382 | } 383 | 384 | if os.Getuid() != 0 { 385 | fmt.Fprintln(os.Stderr, "This command must be run as root") 386 | os.Exit(1) 387 | } 388 | 389 | app := cli.NewApp() 390 | app.Commands = []cli.Command{ 391 | { 392 | Name: "new-interface", 393 | Usage: "Create a new interface", 394 | Action: actionNewInterface, 395 | ArgsUsage: "[--subnet_filter=k,v] [security_group_ids...]", 396 | Flags: []cli.Flag{ 397 | cli.StringFlag{ 398 | Name: "subnet_filter", 399 | Usage: "Comma separated key=value filters to restrict subnets", 400 | }, 401 | cli.Int64Flag{ 402 | Name: "ip_batch_size", 403 | Usage: "Number of ips to allocate on the interface. Specify 0 to max out the interface.", 404 | Value: 1, 405 | }, 406 | }, 407 | }, 408 | { 409 | Name: "remove-interface", 410 | Usage: "Remove an existing interface", 411 | Action: actionRemoveInterface, 412 | ArgsUsage: "[interface_id...]", 413 | }, 414 | { 415 | Name: "deallocate", 416 | Usage: "Deallocate a private IP", 417 | Action: actionDeallocate, 418 | ArgsUsage: "[ip...]", 419 | }, 420 | { 421 | Name: "allocate-first-available", 422 | Usage: "Allocate a private IP on the first available interface", 423 | Action: actionAllocate, 424 | Flags: []cli.Flag{ 425 | cli.IntFlag{ 426 | Name: "index", 427 | }, 428 | cli.Int64Flag{ 429 | Name: "ip_batch_size", 430 | Usage: "Number of ips to allocate on the interface. Specify 0 to max out the interface.", 431 | Value: 1, 432 | }, 433 | }, 434 | }, 435 | { 436 | Name: "free-ips", 437 | Usage: "List all currently unassigned AWS IP addresses", 438 | Action: actionFreeIps, 439 | }, 440 | { 441 | Name: "eniif", 442 | Usage: "List all ENI interfaces and their setup with addresses", 443 | Action: actionEniIf, 444 | }, 445 | { 446 | Name: "addr", 447 | Usage: "List all bound IP addresses", 448 | Action: actionAddr, 449 | }, 450 | { 451 | Name: "subnets", 452 | Usage: "Show available subnets for this host", 453 | Action: actionSubnets, 454 | }, 455 | { 456 | Name: "limits", 457 | Usage: "Display limits for ENI for this instance type", 458 | Action: actionLimits, 459 | }, 460 | { 461 | Name: "maxpods", 462 | Usage: "Return a single number specifying the maximum number of pod addresses that can be used on this instance. Limit maximum with --max", 463 | Action: actionMaxPods, 464 | Flags: []cli.Flag{ 465 | cli.IntFlag{Name: "max"}, 466 | }, 467 | }, 468 | { 469 | Name: "bugs", 470 | Usage: "Show any bugs associated with this instance", 471 | Action: actionBugs, 472 | }, 473 | { 474 | Name: "vpccidr", 475 | Usage: "Show the VPC CIDRs associated with current interfaces", 476 | Action: actionVpcCidr, 477 | }, 478 | { 479 | Name: "vpcpeercidr", 480 | Usage: "Show the peered VPC CIDRs associated with current interfaces", 481 | Action: actionVpcPeerCidr, 482 | }, 483 | { 484 | Name: "registry-list", 485 | Usage: "List all known free IPs in the internal registry", 486 | Action: actionRegistryList, 487 | }, 488 | { 489 | Name: "registry-gc", 490 | Usage: "Free all IPs that have remained unused for a given time interval", 491 | Action: actionRegistryGc, 492 | Flags: []cli.Flag{ 493 | cli.DurationFlag{ 494 | Name: "free-after", 495 | Value: 0 * time.Second, 496 | }, 497 | cli.IntFlag{ 498 | Name: "max-reap", 499 | Value: -1, 500 | Usage: "Max number of ips to reap on a single run. -1 reaps all unused IPs", 501 | }, 502 | }, 503 | }, 504 | } 505 | app.Version = version 506 | app.Copyright = "(c) 2017-2018 Lyft Inc." 507 | app.Usage = "Interface with ENI adapters and CNI bindings for those" 508 | err := app.Run(os.Args) 509 | if err != nil { 510 | fmt.Fprintf(os.Stderr, "Error: %s", err) 511 | os.Exit(1) 512 | } 513 | } 514 | -------------------------------------------------------------------------------- /cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | // TestFilterBuildNil checks the empty string input 8 | func TestFilterBuildNil(t *testing.T) { 9 | ret, err := filterBuild("") 10 | if ret != nil && err != nil { 11 | t.Errorf("Nil input wasn't nil") 12 | } 13 | } 14 | 15 | // TestFilterBuildSingle tests a single filter 16 | func TestFilterBuildSingle(t *testing.T) { 17 | ret, err := filterBuild("foo=bar") 18 | if err != nil { 19 | t.Errorf("Error returned") 20 | } 21 | if v, ok := ret["foo"]; !ok || v != "bar" { 22 | t.Errorf("Invalid return from filter") 23 | } 24 | } 25 | 26 | // TestFilterBuildMulti tests multiple filters 27 | func TestFilterBuildMulti(t *testing.T) { 28 | ret, err := filterBuild("foo=bar,err=frr") 29 | if err != nil { 30 | t.Errorf("Error returned") 31 | } 32 | if v, ok := ret["foo"]; !ok || v != "bar" { 33 | t.Errorf("Invalid return from filter - no foo") 34 | } 35 | if v, ok := ret["err"]; !ok || v != "frr" { 36 | t.Errorf("Invalid return from filter - no frr") 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package cniipvlanvpck8s is the base of a CNI driver to provision addresses across multiple Amazon AWS Elastic 2 | // Network Interfaces, designed to operate within Kubernetes. It performs all state tracking using existing 3 | // system infrastructure. 4 | package cniipvlanvpck8s 5 | -------------------------------------------------------------------------------- /docs/cni.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | blockdiag 10 | seqdiag { 11 | "CNI 12 | runtime" -> "IPvlan 13 | cni-ipvlan-vpc-k8s" [label = "ADD container"]; 14 | "IPvlan 15 | cni-ipvlan-vpc-k8s" -> "IPAM 16 | cni-ipvlan-vpc-k8s" [label = "allocate IP/ENI"]; 17 | "IPvlan 18 | cni-ipvlan-vpc-k8s" <-- "IPAM 19 | cni-ipvlan-vpc-k8s" [label = "IP, ENI, VPC routes, DNS"]; 20 | "CNI 21 | runtime" <-- "IPvlan 22 | cni-ipvlan-vpc-k8s" [label = "Pod IP, interface name"]; 23 | "CNI 24 | runtime" -> "Unnumbered PtP 25 | cni-ipvlan-vpc-k8s" [label = "ADD container(prevResult)"]; 26 | "CNI 27 | runtime" <-- "Unnumbered PtP 28 | cni-ipvlan-vpc-k8s" [label = "prevResult"]; 29 | } 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | CNI 49 | runtime 50 | 51 | IPvlan 52 | cni-ipvlan-vpc-k8s 53 | 54 | IPAM 55 | cni-ipvlan-vpc-k8s 56 | 57 | Unnumbered PtP 58 | cni-ipvlan-vpc-k8s 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | ADD container 72 | allocate IP/ENI 73 | IP, ENI, VPC routes, DNS 74 | Pod IP, interface name 75 | ADD container(prevResult) 76 | prevResult 77 | 78 | -------------------------------------------------------------------------------- /docs/internet-egress.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | blockdiag 10 | seqdiag { 11 | 12 | === Transmit === 13 | Pod -> veth0 [label = "TX pod namespace"]; 14 | veth0 -> vethX [label = "TX default namespace"]; 15 | vethX -> "boot ENI" [label = "forward to eth0"]; 16 | "boot ENI" -> Internet [label = "source NAT"]; 17 | === Receive === 18 | "boot ENI" <-- Internet; 19 | vethX <-- "boot ENI" [label = "remove NAT, forward to Pod"]; 20 | veth0 <-- vethX [label = "RX default namespace"]; 21 | Pod <-- veth0 [label = "RX pod namespace"]; 22 | } 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | Pod 46 | 47 | veth0 48 | 49 | vethX 50 | 51 | boot ENI 52 | 53 | Internet 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | TX pod namespace 71 | TX default namespace 72 | forward to eth0 73 | source NAT 74 | remove NAT, forward to Pod 75 | RX default namespace 76 | RX pod namespace 77 | 78 | 79 | 80 | 81 | 82 | Transmit 83 | 84 | 85 | 86 | 87 | 88 | Receive 89 | 90 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lyft/cni-ipvlan-vpc-k8s 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/Microsoft/go-winio v0.4.11 7 | github.com/alecthomas/units v0.0.0-20190910110746-680d30ca3117 // indirect 8 | github.com/aws/aws-sdk-go v1.29.27 9 | github.com/containernetworking/cni v0.7.1 10 | github.com/containernetworking/plugins v0.8.5 11 | github.com/coreos/go-iptables v0.4.5 12 | github.com/docker/distribution v2.6.2+incompatible 13 | github.com/docker/docker v1.13.1 14 | github.com/docker/go-connections v0.4.0 15 | github.com/docker/go-units v0.3.3 16 | github.com/go-ini/ini v1.39.0 17 | github.com/golangci/golangci-lint v1.18.0 // indirect 18 | github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect 19 | github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 20 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af 21 | github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443 22 | github.com/pkg/errors v0.9.1 23 | github.com/urfave/cli v1.20.0 24 | github.com/vishvananda/netlink v1.0.0 25 | github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc 26 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 27 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f 28 | gopkg.in/alecthomas/gometalinter.v2 v2.0.12 // indirect 29 | gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c // indirect 30 | ) 31 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= 4 | github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= 5 | github.com/OpenPeeDeeP/depguard v1.0.0 h1:k9QF73nrHT3nPLz3lu6G5s+3Hi8Je36ODr1F5gjAXXM= 6 | github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= 7 | github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= 8 | github.com/alecthomas/units v0.0.0-20190910110746-680d30ca3117 h1:aUo+WrWZtRRfc6WITdEKzEczFRlEpfW15NhNeLRc17U= 9 | github.com/alecthomas/units v0.0.0-20190910110746-680d30ca3117/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 10 | github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= 11 | github.com/aws/aws-sdk-go v1.28.1 h1:aWBD5EJrmGFuHFn9ZdaHqWWZGZYQ5Gzb3j9G0RppLpY= 12 | github.com/aws/aws-sdk-go v1.28.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 13 | github.com/aws/aws-sdk-go v1.29.27 h1:4A53lDDGtk4TvnXFzvcOO3Vx3tDqEPfwvChhhxTPN/M= 14 | github.com/aws/aws-sdk-go v1.29.27/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= 15 | github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= 16 | github.com/containernetworking/cni v0.6.0 h1:FXICGBZNMtdHlW65trpoHviHctQD3seWhRRcqp2hMOU= 17 | github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= 18 | github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= 19 | github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= 20 | github.com/containernetworking/plugins v0.7.4 h1:ugkuXfg1Pdzm54U5DGMzreYIkZPSCmSq4rm5TIXVICA= 21 | github.com/containernetworking/plugins v0.7.4/go.mod h1:dagHaAhNjXjT9QYOklkKJDGaQPTg4pf//FrUcJeb7FU= 22 | github.com/containernetworking/plugins v0.8.5 h1:pCvEMrFf7yzJI8+/D/7jkvE96KD52b7/Eu+jpahihy8= 23 | github.com/containernetworking/plugins v0.8.5/go.mod h1:UZ2539umj8djuRQmBxuazHeJbYrLV8BSBejkk+she6o= 24 | github.com/coreos/go-iptables v0.4.0 h1:wh4UbVs8DhLUbpyq97GLJDKrQMjEDD63T1xE4CrsKzQ= 25 | github.com/coreos/go-iptables v0.4.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= 26 | github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38= 27 | github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= 28 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 29 | github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= 30 | github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= 31 | github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= 32 | github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= 33 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 34 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 35 | github.com/docker/distribution v2.6.2+incompatible h1:4FI6af79dfCS/CYb+RRtkSHw3q1L/bnDjG1PcPZtQhM= 36 | github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 37 | github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= 38 | github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 39 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= 40 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 41 | github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= 42 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 43 | github.com/fatih/color v1.6.0 h1:66qjqZk8kalYAvDRtM1AdAJQI0tj4Wrue3Eq3B3pmFU= 44 | github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 45 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 46 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 47 | github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540 h1:djv/qAomOVj8voCHt0M0OYwR/4vfDq1zNKSPKjJCexs= 48 | github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= 49 | github.com/go-ini/ini v1.39.0 h1:/CyW/jTlZLjuzy52jc1XnhJm6IUKEuunpJFpecywNeI= 50 | github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 51 | github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= 52 | github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= 53 | github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= 54 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 55 | github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= 56 | github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= 57 | github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= 58 | github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= 59 | github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= 60 | github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= 61 | github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= 62 | github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= 63 | github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= 64 | github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= 65 | github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= 66 | github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= 67 | github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= 68 | github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= 69 | github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= 70 | github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= 71 | github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= 72 | github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= 73 | github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= 74 | github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= 75 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= 76 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 77 | github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= 78 | github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= 79 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 80 | github.com/golang/mock v1.0.0 h1:HzcpUG60pfl43n9d2qbdi/3l1uKpAmxlfWEPWtV/QxM= 81 | github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 82 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 83 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 84 | github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= 85 | github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= 86 | github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= 87 | github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= 88 | github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= 89 | github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= 90 | github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= 91 | github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= 92 | github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c h1:/7detzz5stiXWPzkTlPTzkBEIIE4WGpppBJYjKqBiPI= 93 | github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= 94 | github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= 95 | github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= 96 | github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= 97 | github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= 98 | github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU= 99 | github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= 100 | github.com/golangci/golangci-lint v1.18.0 h1:XmQgfcLofSG/6AsQuQqmLizB+3GggD+o6ObBG9L+VMM= 101 | github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= 102 | github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547 h1:fUdgm/BdKvwOHxg5AhNbkNRp2mSy8sxTXyBVs/laQHo= 103 | github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= 104 | github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= 105 | github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= 106 | github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE= 107 | github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= 108 | github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= 109 | github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= 110 | github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= 111 | github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= 112 | github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= 113 | github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= 114 | github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= 115 | github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= 116 | github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= 117 | github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= 118 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 119 | github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= 120 | github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= 121 | github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= 122 | github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= 123 | github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno= 124 | github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= 125 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 126 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 127 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 128 | github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ= 129 | github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= 130 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= 131 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 132 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 133 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 134 | github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= 135 | github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= 136 | github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= 137 | github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= 138 | github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= 139 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 140 | github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 141 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 142 | github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 143 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 144 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 145 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 146 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 147 | github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 148 | github.com/magiconair/properties v1.7.6 h1:U+1DqNen04MdEPgFiIwdOUiqZ8qPa37xgogX/sd3+54= 149 | github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 150 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 151 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 152 | github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= 153 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 154 | github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= 155 | github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= 156 | github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= 157 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 158 | github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= 159 | github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 h1:+MZW2uvHgN8kYvksEN3f7eFL2wpzk0GxmlFsMybWc7E= 160 | github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 161 | github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= 162 | github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= 163 | github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw= 164 | github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= 165 | github.com/nicksnyder/go-i18n v2.0.2+incompatible h1:Xt6dluut3s2zBUha8/3sj6atWMQbFioi9OMqUGH9khg= 166 | github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443 h1:+2OJrU8cmOstEoh0uQvYemRGVH1O6xtO2oANUWHFnP0= 167 | github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443/go.mod h1:JbxfV1Iifij2yhRjXai0oFrbpxszXHRx1E5RuM26o4Y= 168 | github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 169 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 170 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 171 | github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 172 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 173 | github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 174 | github.com/pelletier/go-toml v1.1.0 h1:cmiOvKzEunMsAxyhXSzpL5Q1CRKpVv0KQsnAIcSEVYM= 175 | github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 176 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 177 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 178 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 179 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 180 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 181 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 182 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 183 | github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= 184 | github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 185 | github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 186 | github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= 187 | github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= 188 | github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 189 | github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= 190 | github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= 191 | github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= 192 | github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= 193 | github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 194 | github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 195 | github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= 196 | github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= 197 | github.com/spf13/afero v1.1.0 h1:bopulORc2JeYaxfHLvJa5NzxviA9PoWhpiiJkru7Ji4= 198 | github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 199 | github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= 200 | github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= 201 | github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE= 202 | github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 203 | github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig= 204 | github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 205 | github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= 206 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 207 | github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso= 208 | github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= 209 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 210 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 211 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 212 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 213 | github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0= 214 | github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= 215 | github.com/ultraware/funlen v0.0.1 h1:UeC9tpM4wNWzUJfan8z9sFE4QCzjjzlCZmuJN+aOkH0= 216 | github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= 217 | github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= 218 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 219 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 220 | github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= 221 | github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= 222 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 223 | github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= 224 | github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM= 225 | github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= 226 | github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4= 227 | github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= 228 | golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 229 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 230 | golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a h1:YX8ljsm6wXlHZO+aRz9Exqr0evNhKRNe5K/gi+zKh4U= 231 | golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 232 | golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 233 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 234 | golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 235 | golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 236 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= 237 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 238 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 239 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 240 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 241 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= 242 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 243 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 244 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 245 | golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 246 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 247 | golang.org/x/sys v0.0.0-20181119195503-ec83556a53fe h1:I5KvcSfxR/TkvFksuALBTCS44kh6MaPO1rHR9vT0iQQ= 248 | golang.org/x/sys v0.0.0-20181119195503-ec83556a53fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 249 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 250 | golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= 251 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 252 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= 253 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 254 | golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 255 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 256 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 257 | golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 258 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 259 | golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 260 | golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 261 | golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 262 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 263 | golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 264 | golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 265 | golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 266 | golang.org/x/tools v0.0.0-20190909030654-5b82db07426d h1:PhtdWYteEBebOX7KXm4qkIAVSUTHQ883/2hRB92r9lk= 267 | golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 268 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 269 | gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= 270 | gopkg.in/alecthomas/gometalinter.v2 v2.0.12 h1:/xBWwtjmOmVxn8FXfIk9noV8m2E2Id9jFfUY/Mh9QAI= 271 | gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= 272 | gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c h1:vTxShRUnK60yd8DZU+f95p1zSLj814+5CuEh7NjF2/Y= 273 | gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= 274 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 275 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 276 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 277 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 278 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= 279 | gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 280 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 281 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 282 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 283 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 284 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 285 | mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= 286 | mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= 287 | mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= 288 | mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= 289 | mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34 h1:duVSyluuJA+u0BnkcLR01smoLrGgDTfWt5c8ODYG8fU= 290 | mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= 291 | sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= 292 | sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= 293 | -------------------------------------------------------------------------------- /lib/jsontime.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | ) 7 | 8 | // JSONTime is a RFC3339 encoded time with JSON marshallers 9 | type JSONTime struct { 10 | time.Time 11 | } 12 | 13 | // MarshalJSON marshals a JSONTime to an RFC3339 string 14 | func (j *JSONTime) MarshalJSON() ([]byte, error) { 15 | return json.Marshal(j.Time.Format(time.RFC3339)) 16 | } 17 | 18 | // UnmarshalJSON unmarshals a JSONTime to a time.Time 19 | func (j *JSONTime) UnmarshalJSON(js []byte) error { 20 | var rawString string 21 | err := json.Unmarshal(js, &rawString) 22 | if err != nil { 23 | return err 24 | } 25 | t, err := time.Parse(time.RFC3339, rawString) 26 | if err != nil { 27 | return err 28 | } 29 | j.Time = t 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /lib/jsontime_test.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | type Foo struct { 10 | TheTime JSONTime `json:"time"` 11 | } 12 | 13 | func TestJSONTime_MarshalJSON(t *testing.T) { 14 | input := Foo{JSONTime{time.Date(2017, 1, 1, 1, 1, 0, 0, time.UTC)}} 15 | output, err := json.Marshal(&input) 16 | if err != nil { 17 | t.Error(err) 18 | } 19 | if string(output) != `{"time":"2017-01-01T01:01:00Z"}` { 20 | t.Error(string(output)) 21 | } 22 | } 23 | 24 | func TestJSONTime_UnmarshalJSON(t *testing.T) { 25 | input := []byte(`{"time":"2017-01-01T01:01:00Z"}`) 26 | var foo Foo 27 | err := json.Unmarshal(input, &foo) 28 | if err != nil { 29 | t.Error(err) 30 | } 31 | expected := Foo{JSONTime{time.Date(2017, 1, 1, 1, 1, 0, 0, time.UTC)}} 32 | if !foo.TheTime.Time.Equal(expected.TheTime.Time) { 33 | t.Errorf("Times were not equal: %v %v %v", foo.TheTime.Time, expected.TheTime.Time, foo) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lib/lock.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "github.com/nightlyone/lockfile" 10 | ) 11 | 12 | // LockfileRun wraps execution of a specified function around a file lock 13 | func LockfileRun(run func() error) error { 14 | lock, err := lockfile.New(filepath.Join(os.TempDir(), "cni-ipvlan-vpc-k8s.lock")) 15 | if err != nil { 16 | return err 17 | } 18 | tries := 1000 19 | 20 | for { 21 | tries-- 22 | if tries <= 0 { 23 | return fmt.Errorf("Lockfile not acquired, aborting") 24 | } 25 | 26 | err = lock.TryLock() 27 | if err == nil { 28 | break 29 | } else if err == lockfile.ErrBusy { 30 | time.Sleep(100 * time.Millisecond) 31 | } else if err == lockfile.ErrNotExist { 32 | time.Sleep(100 * time.Millisecond) 33 | } else { 34 | return err 35 | } 36 | } 37 | 38 | defer func() { _ = lock.Unlock() }() 39 | return run() 40 | } 41 | -------------------------------------------------------------------------------- /nl/desc.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/containernetworking/plugins/pkg/ns" 11 | "github.com/vishvananda/netlink" 12 | ) 13 | 14 | // BoundIP contains an IPNet / Label pair 15 | type BoundIP struct { 16 | *net.IPNet 17 | Label string 18 | } 19 | 20 | func getIpsOnHandle(handle *netlink.Handle) ([]BoundIP, error) { 21 | foundIps := []BoundIP{} 22 | 23 | links, err := handle.LinkList() 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | for _, link := range links { 29 | addrs, err := handle.AddrList(link, netlink.FAMILY_V4) 30 | if err != nil { 31 | return nil, err 32 | } 33 | for _, addr := range addrs { 34 | ip := *addr.IPNet 35 | found := BoundIP{ 36 | &ip, 37 | addr.Label, 38 | } 39 | foundIps = append(foundIps, found) 40 | } 41 | 42 | } 43 | return foundIps, nil 44 | } 45 | 46 | // GetIPs returns IPs allocated to interfaces, in all namespaces 47 | // TODO: Remove addresses on control plane interfaces, filters 48 | func GetIPs() ([]BoundIP, error) { 49 | 50 | var namespaces []string 51 | 52 | files, err := ioutil.ReadDir("/var/run/netns/") 53 | if err == nil { 54 | for _, file := range files { 55 | namespaces = append(namespaces, 56 | filepath.Join("/var/run/netns", file.Name())) 57 | } 58 | } 59 | 60 | // Check for running docker containers 61 | containers, err := runningDockerContainers() 62 | if err == nil { 63 | dockerNamespaces := dockerNetworkNamespaces(containers) 64 | namespaces = append(namespaces, dockerNamespaces...) 65 | } 66 | 67 | // First get all the IPs in the main namespace 68 | handle, err := netlink.NewHandle() 69 | if err != nil { 70 | return nil, err 71 | } 72 | foundIps, err := getIpsOnHandle(handle) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | // Enter each namesapce, get handles 78 | for _, nsPath := range namespaces { 79 | err := ns.WithNetNSPath(nsPath, func(_ ns.NetNS) error { 80 | 81 | handle, err = netlink.NewHandle() 82 | if err != nil { 83 | return err 84 | } 85 | defer handle.Delete() 86 | 87 | newIps, err := getIpsOnHandle(handle) 88 | foundIps = append(foundIps, newIps...) 89 | return err 90 | }) 91 | if err != nil { 92 | fmt.Fprintf(os.Stderr, "Enumerating namespace failure %v", err) 93 | } 94 | } 95 | 96 | return foundIps, nil 97 | } 98 | -------------------------------------------------------------------------------- /nl/doc.go: -------------------------------------------------------------------------------- 1 | // Package nl provides wrappers for netlink functionality and adding interfaces 2 | package nl 3 | -------------------------------------------------------------------------------- /nl/docker.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/docker/docker/api/types" 7 | "github.com/docker/docker/client" 8 | "golang.org/x/net/context" 9 | ) 10 | 11 | // With a heavy heart and deep internal sadness, we support Docker 12 | func runningDockerContainers() (containerIDs []string, err error) { 13 | cli, err := client.NewEnvClient() 14 | if err != nil { 15 | return nil, err 16 | } 17 | defer cli.Close() 18 | 19 | containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{}) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | for _, container := range containers { 25 | containerIDs = append(containerIDs, container.ID) 26 | } 27 | return containerIDs, nil 28 | } 29 | 30 | func dockerNetworkNamespace(containerID string) (string, error) { 31 | cli, err := client.NewEnvClient() 32 | if err != nil { 33 | return "", err 34 | } 35 | defer cli.Close() 36 | r, err := cli.ContainerInspect(context.Background(), containerID) 37 | if err != nil { 38 | return "", nil 39 | } 40 | 41 | if r.State.Pid == 0 { 42 | // Container has exited 43 | return "", fmt.Errorf("Container has exited %v", containerID) 44 | } 45 | return fmt.Sprintf("/proc/%v/ns/net", r.State.Pid), nil 46 | } 47 | 48 | // Retrieve all namespaces from all running docker containers 49 | func dockerNetworkNamespaces(containerIDs []string) (namespaces []string) { 50 | for _, containerID := range containerIDs { 51 | cid, err := dockerNetworkNamespace(containerID) 52 | if err == nil && len(cid) > 0 { 53 | namespaces = append(namespaces, cid) 54 | } 55 | } 56 | return 57 | } 58 | -------------------------------------------------------------------------------- /nl/down.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/vishvananda/netlink" 7 | ) 8 | 9 | // DownInterface Takes down a single interface 10 | func DownInterface(name string) error { 11 | link, err := netlink.LinkByName(name) 12 | 13 | if err != nil { 14 | return err 15 | } 16 | 17 | if err := netlink.LinkSetDown(link); err != nil { 18 | fmt.Printf("Unable to set interface %v down", name) 19 | return err 20 | } 21 | 22 | return nil 23 | } 24 | 25 | // RemoveInterface complete removes the interface 26 | func RemoveInterface(name string) error { 27 | link, err := netlink.LinkByName(name) 28 | 29 | if err != nil { 30 | return err 31 | } 32 | 33 | if err := netlink.LinkDel(link); err != nil { 34 | fmt.Printf("Unable to remove interface %v", name) 35 | return err 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /nl/down_test.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/vishvananda/netlink" 8 | ) 9 | 10 | func TestDownInterface(t *testing.T) { 11 | if os.Getuid() != 0 { 12 | t.Skip("Test requires root or network capabilities - skipped") 13 | return 14 | } 15 | 16 | CreateTestInterface(t, "lyft3") 17 | defer func() { _ = RemoveInterface("lyft3") }() 18 | 19 | if err := UpInterface("lyft3"); err != nil { 20 | t.Fatalf("Failed to UpInterface lyft3: %v", err) 21 | } 22 | 23 | if err := DownInterface("lyft3"); err != nil { 24 | t.Fatalf("Failed to DownInterface lyft3: %v", err) 25 | } 26 | } 27 | 28 | func TestRemoveInterface(t *testing.T) { 29 | if os.Getuid() != 0 { 30 | t.Skip("Test requires root or network capabilities - skipped") 31 | return 32 | } 33 | 34 | CreateTestInterface(t, "lyft4") 35 | 36 | if err := RemoveInterface("lyft4"); err != nil { 37 | t.Fatalf("Failed to RemoveInterface lyft4: %v", err) 38 | } 39 | 40 | // This link should not exist, this call should fail 41 | link, _ := netlink.LinkByName("lyft4") 42 | 43 | if link != nil { 44 | t.Fatal("Failed to RemoveInterface lyft4") 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /nl/mtu.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "github.com/vishvananda/netlink" 5 | ) 6 | 7 | // GetMtu gets the current MTU for an interface 8 | func GetMtu(name string) (int, error) { 9 | link, err := netlink.LinkByName(name) 10 | if err != nil { 11 | return 0, err 12 | } 13 | return link.Attrs().MTU, nil 14 | } 15 | 16 | // SetMtu sets the MTU of an interface 17 | func SetMtu(name string, mtu int) error { 18 | link, err := netlink.LinkByName(name) 19 | if err != nil { 20 | return err 21 | } 22 | return netlink.LinkSetMTU(link, mtu) 23 | } 24 | -------------------------------------------------------------------------------- /nl/up.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/vishvananda/netlink" 9 | ) 10 | 11 | const interfaceSettleWaitTime = 100 * time.Millisecond 12 | const interfaceSettleDeadline = 20 * time.Second 13 | 14 | // UpInterface brings up an interface by name 15 | func UpInterface(name string) error { 16 | link, err := netlink.LinkByName(name) 17 | if err != nil { 18 | return err 19 | } 20 | return netlink.LinkSetUp(link) 21 | } 22 | 23 | // UpInterfacePoll waits until an interface can be resolved by netlink and then call up on the interface. 24 | func UpInterfacePoll(name string) error { 25 | for start := time.Now(); time.Since(start) <= interfaceSettleDeadline; time.Sleep(interfaceSettleWaitTime) { 26 | err := UpInterface(name) 27 | if err == nil { 28 | return nil 29 | } 30 | _, err = fmt.Fprintf(os.Stderr, "Failing to enumerate %v due to %v\n", name, err) 31 | if err != nil { 32 | panic(err) 33 | } 34 | } 35 | return fmt.Errorf("Interface was not found after setting time") 36 | } 37 | -------------------------------------------------------------------------------- /nl/up_test.go: -------------------------------------------------------------------------------- 1 | package nl 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/vishvananda/netlink" 8 | ) 9 | 10 | func CreateTestInterface(t *testing.T, name string) { 11 | lyftBridge := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{ 12 | TxQLen: -1, 13 | Name: name, 14 | }} 15 | 16 | err := netlink.LinkAdd(lyftBridge) 17 | if err != nil { 18 | t.Errorf("Could not add %s: %v", lyftBridge.Name, err) 19 | err = RemoveInterface(lyftBridge.Name) 20 | if err != nil { 21 | t.Errorf("Failed to remove interface %s: %s", lyftBridge.Name, err) 22 | } 23 | } 24 | 25 | lyft1, _ := netlink.LinkByName(name) 26 | err = netlink.LinkSetMaster(lyft1, lyftBridge) 27 | if err != nil { 28 | t.Logf("Failed to set link master: %s", err) 29 | } 30 | } 31 | 32 | func TestUpInterface(t *testing.T) { 33 | if os.Getuid() != 0 { 34 | t.Skip("Test requires root or network capabilities - skipped") 35 | return 36 | } 37 | 38 | CreateTestInterface(t, "lyft1") 39 | defer func() { _ = RemoveInterface("lyft1") }() 40 | 41 | if err := UpInterface("lyft1"); err != nil { 42 | t.Fatalf("Failed to UpInterface %v", err) 43 | } 44 | } 45 | 46 | func TestUpInterfacePoll(t *testing.T) { 47 | if os.Getuid() != 0 { 48 | t.Skip("Test requires root or network capabilities - skipped") 49 | return 50 | } 51 | 52 | CreateTestInterface(t, "lyft2") 53 | defer func() { _ = RemoveInterface("lyft2") }() 54 | 55 | if err := UpInterfacePoll("lyft2"); err != nil { 56 | t.Fatalf("Failed to failed to stand up interface lyft2") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /plugin/ipam/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 CNI authors 2 | // Copyright 2017 Lyft, Inc. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // This is a sample chained plugin that supports multiple CNI versions. It 17 | // parses prevResult according to the cniVersion 18 | package main 19 | 20 | import ( 21 | "encoding/json" 22 | "fmt" 23 | "net" 24 | "runtime" 25 | "time" 26 | 27 | "github.com/containernetworking/cni/pkg/skel" 28 | "github.com/containernetworking/cni/pkg/types" 29 | "github.com/containernetworking/cni/pkg/types/current" 30 | "github.com/containernetworking/cni/pkg/version" 31 | "github.com/containernetworking/plugins/pkg/ns" 32 | "github.com/vishvananda/netlink" 33 | 34 | "github.com/lyft/cni-ipvlan-vpc-k8s/aws" 35 | "github.com/lyft/cni-ipvlan-vpc-k8s/lib" 36 | "github.com/lyft/cni-ipvlan-vpc-k8s/nl" 37 | ) 38 | 39 | // PluginConf contains configuration parameters 40 | type PluginConf struct { 41 | Name string `json:"name"` 42 | CNIVersion string `json:"cniVersion"` 43 | SecGroupIds []string `json:"secGroupIds"` 44 | SubnetTags map[string]string `json:"subnetTags"` 45 | IfaceIndex int `json:"interfaceIndex"` 46 | SkipDeallocation bool `json:"skipDeallocation"` 47 | RouteToVPCPeers bool `json:"routeToVpcPeers"` 48 | ReuseIPWait int `json:"reuseIPWait"` 49 | IPBatchSize int64 `json:"ipBatchSize"` 50 | RouteToCidrs []string `json:"routeToCidrs"` 51 | } 52 | 53 | func init() { 54 | // this ensures that main runs only on main thread (thread group leader). 55 | // since namespace ops (unshare, setns) are done for a single thread, we 56 | // must ensure that the goroutine does not jump from OS thread to thread 57 | runtime.LockOSThread() 58 | } 59 | 60 | // parseConfig parses the supplied configuration from stdin. 61 | func parseConfig(stdin []byte) (*PluginConf, error) { 62 | conf := PluginConf{ 63 | ReuseIPWait: 60, // default 60 second wait 64 | IPBatchSize: 1, // default 1 (backward compatibility) 65 | } 66 | 67 | if err := json.Unmarshal(stdin, &conf); err != nil { 68 | return nil, fmt.Errorf("failed to parse network configuration: %v", err) 69 | } 70 | 71 | if conf.SecGroupIds == nil { 72 | return nil, fmt.Errorf("secGroupIds must be specified") 73 | } 74 | 75 | return &conf, nil 76 | } 77 | 78 | // cmdAdd is called for ADD requests 79 | func cmdAdd(args *skel.CmdArgs) error { 80 | conf, err := parseConfig(args.StdinData) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | var alloc *aws.AllocationResult 86 | registry := &aws.Registry{} 87 | 88 | // Try to find a free IP first - possibly from a broken 89 | // container, or torn down namespace. IP must also be at least 90 | // conf.ReuseIPWait seconds old in the registry to be 91 | // considered for use. 92 | free, err := aws.FindFreeIPsAtIndex(conf.IfaceIndex, true) 93 | if err == nil || len(free) > 0 { 94 | registryFreeIPs, err := registry.TrackedBefore(time.Now().Add(time.Duration(-conf.ReuseIPWait) * time.Second)) 95 | if err == nil && len(registryFreeIPs) > 0 { 96 | loop: 97 | for _, freeAlloc := range free { 98 | for _, freeRegistry := range registryFreeIPs { 99 | if freeAlloc.IP.Equal(freeRegistry) { 100 | alloc = freeAlloc 101 | // update timestamp 102 | err := registry.TrackIP(freeRegistry) 103 | if err != nil { 104 | return fmt.Errorf("failed to track ip: %s", err) 105 | } 106 | break loop 107 | } 108 | } 109 | } 110 | } 111 | } 112 | 113 | // No free IPs available for use, so let's allocate one 114 | if alloc == nil { 115 | // allocate IPs on an available interface 116 | allocs, err := aws.DefaultClient.AllocateIPsFirstAvailableAtIndex(conf.IfaceIndex, conf.IPBatchSize) 117 | if err == nil || len(allocs) > 0 { 118 | alloc = allocs[0] 119 | } else { 120 | // failed, so attempt to add an IP to a new interface 121 | newIf, err := aws.DefaultClient.NewInterface(conf.SecGroupIds, conf.SubnetTags, conf.IPBatchSize) 122 | if err != nil || len(newIf.IPv4s) < 1 { 123 | return fmt.Errorf("unable to create a new elastic network interface due to %v", 124 | err) 125 | } 126 | // Freshly allocated interfaces will always have at least one valid IP - use 127 | // this IP address. 128 | alloc = &aws.AllocationResult{ 129 | IP: &newIf.IPv4s[0], 130 | Interface: *newIf, 131 | } 132 | } 133 | } 134 | 135 | // Per https://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html 136 | // subnet + 1 is our gateway 137 | // primary cidr + 2 is the dns server 138 | subnetAddr := alloc.Interface.SubnetCidr.IP.To4() 139 | gw := append(subnetAddr[:3], subnetAddr[3]+1) 140 | vpcPrimaryAddr := alloc.Interface.VpcPrimaryCidr.IP.To4() 141 | dns := append(vpcPrimaryAddr[:3], vpcPrimaryAddr[3]+2) 142 | addr := net.IPNet{ 143 | IP: *alloc.IP, 144 | Mask: alloc.Interface.SubnetCidr.Mask, 145 | } 146 | 147 | master := alloc.Interface.LocalName() 148 | 149 | iface := ¤t.Interface{ 150 | Name: master, 151 | } 152 | 153 | // Ensure the master interface is always up 154 | err = nl.UpInterfacePoll(master) 155 | if err != nil { 156 | return fmt.Errorf("unable to bring up interface %v due to %v", 157 | master, err) 158 | } 159 | 160 | ipconfig := ¤t.IPConfig{ 161 | Version: "4", 162 | Address: addr, 163 | Gateway: gw, 164 | Interface: current.Int(0), 165 | } 166 | 167 | result := ¤t.Result{} 168 | rDNS := types.DNS{} 169 | rDNS.Nameservers = append(rDNS.Nameservers, dns.String()) 170 | result.DNS = rDNS 171 | result.IPs = append(result.IPs, ipconfig) 172 | result.Interfaces = append(result.Interfaces, iface) 173 | 174 | cidrs := alloc.Interface.VpcCidrs 175 | if aws.HasBugBrokenVPCCidrs(aws.DefaultClient) { 176 | cidrs, err = aws.DefaultClient.DescribeVPCCIDRs(alloc.Interface.VpcID) 177 | if err != nil { 178 | return fmt.Errorf("Unable to enumerate CIDRs from the AWS API due to a specific meta-data bug %v", err) 179 | } 180 | } 181 | 182 | if conf.RouteToVPCPeers { 183 | peerCidr, err := aws.DefaultClient.DescribeVPCPeerCIDRs(alloc.Interface.VpcID) 184 | if err != nil { 185 | return fmt.Errorf("unable to enumerate peer CIDrs %v", err) 186 | } 187 | cidrs = append(cidrs, peerCidr...) 188 | } 189 | 190 | if conf.RouteToCidrs != nil { 191 | for _, cidr := range conf.RouteToCidrs { 192 | _, parsed, err := net.ParseCIDR(cidr) 193 | if err != nil { 194 | return fmt.Errorf("unable to parse routeToCidrs element %v", err) 195 | } 196 | cidrs = append(cidrs, parsed) 197 | } 198 | } 199 | 200 | // add routes for all VPC cidrs via the subnet gateway 201 | for _, dst := range cidrs { 202 | result.Routes = append(result.Routes, &types.Route{Dst: *dst, GW: gw}) 203 | } 204 | 205 | // remove the IP from the registry just before handing off to ipvlan 206 | err = registry.ForgetIP(*alloc.IP) 207 | if err != nil { 208 | return fmt.Errorf("failed to forget ip: %s", err) 209 | } 210 | 211 | return types.PrintResult(result, conf.CNIVersion) 212 | } 213 | 214 | // cmdDel is called for DELETE requests 215 | func cmdDel(args *skel.CmdArgs) error { 216 | conf, err := parseConfig(args.StdinData) 217 | if err != nil { 218 | return err 219 | } 220 | _ = conf 221 | 222 | var addrs []netlink.Addr 223 | 224 | // enter the namespace to grab the list of IPs 225 | _ = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { 226 | iface, err := netlink.LinkByName(args.IfName) 227 | if err != nil { 228 | return err 229 | } 230 | addrs, err = netlink.AddrList(iface, netlink.FAMILY_V4) 231 | return err 232 | }) 233 | 234 | registry := &aws.Registry{} 235 | for _, addr := range addrs { 236 | if !conf.SkipDeallocation { 237 | // deallocate IPs outside of the namespace so creds are correct 238 | err := aws.DefaultClient.DeallocateIP(&addr.IP) 239 | if err != nil { 240 | return fmt.Errorf("failed to deallocate ip: %s", err) 241 | } 242 | } 243 | // Mark this IP as free in the registry 244 | err := registry.TrackIP(addr.IP) 245 | if err != nil { 246 | return fmt.Errorf("failed to track ip: %s", err) 247 | } 248 | } 249 | 250 | return nil 251 | } 252 | 253 | // cmdCheck is called for CHECK requests 254 | func cmdCheck(args *skel.CmdArgs) error { 255 | // TODO: implement this 256 | return nil 257 | } 258 | 259 | func main() { 260 | run := func() error { 261 | skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.PluginSupports(version.Current()), "ipam") 262 | return nil 263 | } 264 | _ = lib.LockfileRun(run) 265 | } 266 | -------------------------------------------------------------------------------- /plugin/ipvlan/ipvlan.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 CNI authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "encoding/json" 19 | "errors" 20 | "fmt" 21 | "runtime" 22 | 23 | "github.com/containernetworking/cni/pkg/skel" 24 | "github.com/containernetworking/cni/pkg/types" 25 | "github.com/containernetworking/cni/pkg/types/current" 26 | "github.com/containernetworking/cni/pkg/version" 27 | "github.com/containernetworking/plugins/pkg/ip" 28 | "github.com/containernetworking/plugins/pkg/ipam" 29 | "github.com/containernetworking/plugins/pkg/ns" 30 | "github.com/vishvananda/netlink" 31 | ) 32 | 33 | // NetConf contains network configuration parameters 34 | type NetConf struct { 35 | types.NetConf 36 | 37 | // support chaining for master interface and IP decisions 38 | // occurring prior to running ipvlan plugin 39 | RawPrevResult *map[string]interface{} `json:"prevResult"` 40 | PrevResult *current.Result `json:"-"` 41 | 42 | Master string `json:"master"` 43 | Mode string `json:"mode"` 44 | MTU int `json:"mtu"` 45 | } 46 | 47 | const ( 48 | cniAdd = iota 49 | cniDel 50 | ) 51 | 52 | func init() { 53 | // this ensures that main runs only on main thread (thread group leader). 54 | // since namespace ops (unshare, setns) are done for a single thread, we 55 | // must ensure that the goroutine does not jump from OS thread to thread 56 | runtime.LockOSThread() 57 | } 58 | 59 | func loadConf(bytes []byte, cmd int) (*NetConf, string, error) { 60 | n := &NetConf{} 61 | if err := json.Unmarshal(bytes, n); err != nil { 62 | return nil, "", fmt.Errorf("failed to load netconf: %v", err) 63 | } 64 | // Parse previous result 65 | if n.RawPrevResult != nil { 66 | resultBytes, err := json.Marshal(n.RawPrevResult) 67 | if err != nil { 68 | return nil, "", fmt.Errorf("could not serialize prevResult: %v", err) 69 | } 70 | res, err := version.NewResult(n.CNIVersion, resultBytes) 71 | if err != nil { 72 | return nil, "", fmt.Errorf("could not parse prevResult: %v", err) 73 | } 74 | n.RawPrevResult = nil 75 | n.PrevResult, err = current.NewResultFromResult(res) 76 | if err != nil { 77 | return nil, "", fmt.Errorf("could not convert result to current version: %v", err) 78 | } 79 | } 80 | if n.Master == "" && cmd != cniDel { 81 | if n.PrevResult == nil { 82 | return nil, "", fmt.Errorf(`"master" field is required. It specifies the host interface name to virtualize`) 83 | } 84 | if len(n.PrevResult.Interfaces) == 1 && n.PrevResult.Interfaces[0].Name != "" { 85 | n.Master = n.PrevResult.Interfaces[0].Name 86 | } else { 87 | return nil, "", fmt.Errorf("chained master failure. PrevResult lacks a single named interface") 88 | } 89 | } 90 | return n, n.CNIVersion, nil 91 | } 92 | 93 | func modeFromString(s string) (netlink.IPVlanMode, error) { 94 | switch s { 95 | case "", "l2": 96 | return netlink.IPVLAN_MODE_L2, nil 97 | case "l3": 98 | return netlink.IPVLAN_MODE_L3, nil 99 | case "l3s": 100 | return netlink.IPVLAN_MODE_L3S, nil 101 | default: 102 | return 0, fmt.Errorf("unknown ipvlan mode: %q", s) 103 | } 104 | } 105 | 106 | func createIpvlan(conf *NetConf, ifName string, netns ns.NetNS) (*current.Interface, error) { 107 | ipvlan := ¤t.Interface{} 108 | 109 | mode, err := modeFromString(conf.Mode) 110 | if err != nil { 111 | return nil, err 112 | } 113 | 114 | m, err := netlink.LinkByName(conf.Master) 115 | if err != nil { 116 | return nil, fmt.Errorf("failed to lookup master %q: %v", conf.Master, err) 117 | } 118 | 119 | // due to kernel bug we have to create with tmpname or it might 120 | // collide with the name on the host and error out 121 | tmpName, err := ip.RandomVethName() 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | mv := &netlink.IPVlan{ 127 | LinkAttrs: netlink.LinkAttrs{ 128 | MTU: conf.MTU, 129 | Name: tmpName, 130 | ParentIndex: m.Attrs().Index, 131 | Namespace: netlink.NsFd(int(netns.Fd())), 132 | }, 133 | Mode: mode, 134 | } 135 | 136 | if err := netlink.LinkAdd(mv); err != nil { 137 | return nil, fmt.Errorf("failed to create ipvlan: %v", err) 138 | } 139 | 140 | err = netns.Do(func(_ ns.NetNS) error { 141 | err := ip.RenameLink(tmpName, ifName) 142 | if err != nil { 143 | return fmt.Errorf("failed to rename ipvlan to %q: %v", ifName, err) 144 | } 145 | ipvlan.Name = ifName 146 | 147 | // Re-fetch ipvlan to get all properties/attributes 148 | contIpvlan, err := netlink.LinkByName(ipvlan.Name) 149 | if err != nil { 150 | return fmt.Errorf("failed to refetch ipvlan %q: %v", ipvlan.Name, err) 151 | } 152 | ipvlan.Mac = contIpvlan.Attrs().HardwareAddr.String() 153 | ipvlan.Sandbox = netns.Path() 154 | 155 | return nil 156 | }) 157 | if err != nil { 158 | return nil, err 159 | } 160 | 161 | return ipvlan, nil 162 | } 163 | 164 | func cmdAdd(args *skel.CmdArgs) error { 165 | n, cniVersion, err := loadConf(args.StdinData, cniAdd) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | netns, err := ns.GetNS(args.Netns) 171 | if err != nil { 172 | return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) 173 | } 174 | defer netns.Close() 175 | 176 | ipvlanInterface, err := createIpvlan(n, args.IfName, netns) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | var result *current.Result 182 | // Configure iface from PrevResult if we have IPs and an IPAM 183 | // block has not been configured 184 | if n.IPAM.Type == "" && n.PrevResult != nil && len(n.PrevResult.IPs) > 0 { 185 | result = n.PrevResult 186 | } else { 187 | // run the IPAM plugin and get back the config to apply 188 | r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) 189 | if err != nil { 190 | return err 191 | } 192 | // Convert whatever the IPAM result was into the current Result type 193 | result, err = current.NewResultFromResult(r) 194 | if err != nil { 195 | return err 196 | } 197 | 198 | if len(result.IPs) == 0 { 199 | return errors.New("IPAM plugin returned missing IP config") 200 | } 201 | } 202 | for _, ipc := range result.IPs { 203 | // All addresses belong to the ipvlan interface 204 | ipc.Interface = current.Int(0) 205 | } 206 | 207 | result.Interfaces = []*current.Interface{ipvlanInterface} 208 | 209 | err = netns.Do(func(_ ns.NetNS) error { 210 | return ipam.ConfigureIface(args.IfName, result) 211 | }) 212 | if err != nil { 213 | return err 214 | } 215 | 216 | result.DNS = n.DNS 217 | 218 | return types.PrintResult(result, cniVersion) 219 | } 220 | 221 | func cmdDel(args *skel.CmdArgs) error { 222 | n, _, err := loadConf(args.StdinData, cniDel) 223 | if err != nil { 224 | return err 225 | } 226 | 227 | // On chained invocation, IPAM block can be empty 228 | if n.IPAM.Type != "" { 229 | err = ipam.ExecDel(n.IPAM.Type, args.StdinData) 230 | if err != nil { 231 | return err 232 | } 233 | } 234 | 235 | // On chained invocation, Master can be empty 236 | if n.Master == "" { 237 | return nil 238 | } 239 | 240 | if args.Netns == "" { 241 | return nil 242 | } 243 | 244 | // There is a netns so try to clean up. Delete can be called multiple times 245 | // so don't return an error if the device is already removed. 246 | err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { 247 | if err := ip.DelLinkByName(args.IfName); err != nil { 248 | if err != ip.ErrLinkNotFound { 249 | return err 250 | } 251 | } 252 | return nil 253 | }) 254 | 255 | return err 256 | } 257 | 258 | // cmdCheck is called for CHECK requests 259 | func cmdCheck(args *skel.CmdArgs) error { 260 | // TODO: implement this 261 | return nil 262 | } 263 | 264 | func main() { 265 | skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, "ipvlan") 266 | } 267 | -------------------------------------------------------------------------------- /plugin/unnumbered-ptp/unnumbered-ptp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 CNI authors 2 | // Copyright 2017 Lyft Inc. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // This is a sample chained plugin that supports multiple CNI versions. It 16 | // parses prevResult according to the cniVersion 17 | package main 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "math" 23 | "math/rand" 24 | "net" 25 | "os" 26 | "sort" 27 | "strconv" 28 | "time" 29 | 30 | "github.com/containernetworking/cni/pkg/skel" 31 | "github.com/containernetworking/cni/pkg/types" 32 | "github.com/containernetworking/cni/pkg/types/current" 33 | "github.com/containernetworking/cni/pkg/version" 34 | "github.com/containernetworking/plugins/pkg/ip" 35 | "github.com/containernetworking/plugins/pkg/ns" 36 | "github.com/containernetworking/plugins/pkg/utils" 37 | "github.com/containernetworking/plugins/pkg/utils/sysctl" 38 | "github.com/coreos/go-iptables/iptables" 39 | "github.com/j-keck/arping" 40 | "github.com/lyft/cni-ipvlan-vpc-k8s/nl" 41 | "github.com/vishvananda/netlink" 42 | ) 43 | 44 | // constants for full jitter backoff in milliseconds, and for nodeport marks 45 | const ( 46 | maxSleep = 10000 // 10.00s 47 | baseSleep = 20 // 0.02 48 | RPFilterTemplate = "net.ipv4.conf.%s.rp_filter" 49 | podRulePriority = 1024 50 | nodePortRulePriority = 512 51 | ) 52 | 53 | // PluginConf is whatever you expect your configuration json to be. This is whatever 54 | // is passed in on stdin. Your plugin may wish to expose its functionality via 55 | // runtime args, see CONVENTIONS.md in the CNI spec. 56 | type PluginConf struct { 57 | types.NetConf 58 | 59 | // This is the previous result, when called in the context of a chained 60 | // plugin. Because this plugin supports multiple versions, we'll have to 61 | // parse this in two passes. If your plugin is not chained, this can be 62 | // removed (though you may wish to error if a non-chainable plugin is 63 | // chained. 64 | // If you need to modify the result before returning it, you will need 65 | // to actually convert it to a concrete versioned struct. 66 | RawPrevResult *map[string]interface{} `json:"prevResult"` 67 | PrevResult *current.Result `json:"-"` 68 | 69 | IPMasq bool `json:"ipMasq"` 70 | HostInterface string `json:"hostInterface"` 71 | ContainerInterface string `json:"containerInterface"` 72 | MTU int `json:"mtu"` 73 | TableStart int `json:"routeTableStart"` 74 | NodePortMark int `json:"nodePortMark"` 75 | NodePorts string `json:"nodePorts"` 76 | } 77 | 78 | // parseConfig parses the supplied configuration (and prevResult) from stdin. 79 | func parseConfig(stdin []byte) (*PluginConf, error) { 80 | conf := PluginConf{} 81 | 82 | if err := json.Unmarshal(stdin, &conf); err != nil { 83 | return nil, fmt.Errorf("failed to parse network configuration: %v", err) 84 | } 85 | 86 | // Parse previous result. 87 | if conf.RawPrevResult != nil { 88 | resultBytes, err := json.Marshal(conf.RawPrevResult) 89 | if err != nil { 90 | return nil, fmt.Errorf("could not serialize prevResult: %v", err) 91 | } 92 | res, err := version.NewResult(conf.CNIVersion, resultBytes) 93 | if err != nil { 94 | return nil, fmt.Errorf("could not parse prevResult: %v", err) 95 | } 96 | conf.RawPrevResult = nil 97 | conf.PrevResult, err = current.NewResultFromResult(res) 98 | if err != nil { 99 | return nil, fmt.Errorf("could not convert result to current version: %v", err) 100 | } 101 | } 102 | // End previous result parsing 103 | 104 | if conf.HostInterface == "" { 105 | return nil, fmt.Errorf("hostInterface must be specified") 106 | } 107 | 108 | // If the MTU is not set, use the one of the hostInterface 109 | if conf.MTU == 0 { 110 | baseMtu, err := nl.GetMtu(conf.HostInterface) 111 | if err != nil { 112 | return nil, fmt.Errorf("unable to get MTU for hostInterface") 113 | } 114 | conf.MTU = baseMtu 115 | } 116 | 117 | if conf.ContainerInterface == "" { 118 | return nil, fmt.Errorf("containerInterface must be specified") 119 | } 120 | 121 | if conf.NodePorts == "" { 122 | conf.NodePorts = "30000:32767" 123 | } 124 | 125 | if conf.NodePortMark == 0 { 126 | conf.NodePortMark = 0x2000 127 | } 128 | 129 | // start using tables by default at 256 130 | if conf.TableStart == 0 { 131 | conf.TableStart = 256 132 | } 133 | 134 | return &conf, nil 135 | } 136 | 137 | func enableForwarding(ipv4 bool, ipv6 bool) error { 138 | if ipv4 { 139 | err := ip.EnableIP4Forward() 140 | if err != nil { 141 | return fmt.Errorf("Could not enable IPv6 forwarding: %v", err) 142 | } 143 | } 144 | if ipv6 { 145 | err := ip.EnableIP6Forward() 146 | if err != nil { 147 | return fmt.Errorf("Could not enable IPv6 forwarding: %v", err) 148 | } 149 | } 150 | return nil 151 | } 152 | 153 | func setupSNAT(ifName string, comment string) error { 154 | ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) 155 | if err != nil { 156 | return fmt.Errorf("failed to locate iptables: %v", err) 157 | } 158 | rulespec := []string{"-o", ifName, "-j", "MASQUERADE"} 159 | if ipt.HasRandomFully() { 160 | rulespec = append(rulespec, "--random-fully") 161 | } 162 | rulespec = append(rulespec, "-m", "comment", "--comment", comment) 163 | return ipt.AppendUnique("nat", "POSTROUTING", rulespec...) 164 | } 165 | 166 | func findFreeTable(start int) (int, error) { 167 | allocatedTableIDs := make(map[int]bool) 168 | // combine V4 and V6 tables 169 | for _, family := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { 170 | rules, err := netlink.RuleList(family) 171 | if err != nil { 172 | return -1, err 173 | } 174 | for _, rule := range rules { 175 | allocatedTableIDs[rule.Table] = true 176 | } 177 | } 178 | // find first slot that's available for both V4 and V6 usage 179 | for i := start; i < math.MaxUint32; i++ { 180 | if !allocatedTableIDs[i] { 181 | return i, nil 182 | } 183 | } 184 | return -1, fmt.Errorf("failed to find free route table") 185 | } 186 | 187 | func addPolicyRules(veth *net.Interface, ipc *current.IPConfig, routes []*types.Route, tableStart int) error { 188 | table := -1 189 | 190 | // depend on netlink atomicity to win races for table slots on initial route add 191 | sort.Slice(routes, func(i, j int) bool { 192 | return routes[i].Dst.String() < routes[j].Dst.String() 193 | }) 194 | 195 | // try 10 times to write to an empty table slot 196 | for i := 0; i < 10 && table == -1; i++ { 197 | var err error 198 | // jitter looking for an initial free table slot 199 | table, err = findFreeTable(tableStart + rand.Intn(1000)) 200 | if err != nil { 201 | return err 202 | } 203 | 204 | // add routes to the policy routing table 205 | for _, route := range routes { 206 | err := netlink.RouteAdd(&netlink.Route{ 207 | LinkIndex: veth.Index, 208 | Dst: &route.Dst, 209 | Gw: ipc.Address.IP, 210 | Table: table, 211 | }) 212 | if err != nil { 213 | table = -1 214 | break 215 | } 216 | } 217 | 218 | if table == -1 { 219 | // failed to add routes so sleep and try again on a different table 220 | wait := time.Duration(rand.Intn(int(math.Min(maxSleep, 221 | baseSleep*math.Pow(2, float64(i)))))) * time.Millisecond 222 | fmt.Fprintf(os.Stderr, "route table collision, retrying in %v\n", wait) 223 | time.Sleep(wait) 224 | } 225 | } 226 | 227 | // ensure we have a route table selected 228 | if table == -1 { 229 | return fmt.Errorf("failed to add routes to a free table") 230 | } 231 | 232 | // add policy route for traffic originating from a Pod 233 | rule := netlink.NewRule() 234 | rule.IifName = veth.Name 235 | rule.Table = table 236 | rule.Priority = podRulePriority 237 | 238 | err := netlink.RuleAdd(rule) 239 | if err != nil { 240 | return fmt.Errorf("failed to add policy rule %v: %v", rule, err) 241 | } 242 | 243 | return nil 244 | } 245 | 246 | func setupNodePortRule(ifName string, nodePorts string, nodePortMark int) error { 247 | ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) 248 | if err != nil { 249 | return fmt.Errorf("failed to locate iptables: %v", err) 250 | } 251 | 252 | // Create iptables rules to ensure that nodeport traffic is marked 253 | if err := ipt.AppendUnique("mangle", "PREROUTING", "-i", ifName, "-p", "tcp", "--dport", nodePorts, "-j", "CONNMARK", "--set-mark", strconv.Itoa(nodePortMark), "-m", "comment", "--comment", "NodePort Mark"); err != nil { 254 | return err 255 | } 256 | if err := ipt.AppendUnique("mangle", "PREROUTING", "-i", ifName, "-p", "udp", "--dport", nodePorts, "-j", "CONNMARK", "--set-mark", strconv.Itoa(nodePortMark), "-m", "comment", "--comment", "NodePort Mark"); err != nil { 257 | return err 258 | } 259 | if err := ipt.AppendUnique("mangle", "PREROUTING", "-i", "veth+", "-j", "CONNMARK", "--restore-mark", "-m", "comment", "--comment", "NodePort Mark"); err != nil { 260 | return err 261 | } 262 | 263 | // Use loose RP filter on host interface (RP filter does not take mark-based rules into account) 264 | _, err = sysctl.Sysctl(fmt.Sprintf(RPFilterTemplate, ifName), "2") 265 | if err != nil { 266 | return fmt.Errorf("failed to set RP filter to loose for interface %q: %v", ifName, err) 267 | } 268 | 269 | // add policy route for traffic from marked as nodeport 270 | rule := netlink.NewRule() 271 | rule.Mark = nodePortMark 272 | rule.Table = 254 // main table 273 | rule.Priority = nodePortRulePriority 274 | 275 | exists := false 276 | rules, err := netlink.RuleList(netlink.FAMILY_V4) 277 | if err != nil { 278 | return fmt.Errorf("Unable to retrieve IP rules %v", err) 279 | } 280 | 281 | for _, r := range rules { 282 | if r.Table == rule.Table && r.Mark == rule.Mark && r.Priority == rule.Priority { 283 | exists = true 284 | break 285 | } 286 | } 287 | if !exists { 288 | err := netlink.RuleAdd(rule) 289 | if err != nil { 290 | return fmt.Errorf("failed to add policy rule %v: %v", rule, err) 291 | } 292 | } 293 | 294 | return nil 295 | } 296 | 297 | func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, hostAddrs []netlink.Addr, masq, containerIPV4, containerIPV6 bool, k8sIfName string, pr *current.Result) (*current.Interface, *current.Interface, error) { 298 | hostInterface := ¤t.Interface{} 299 | containerInterface := ¤t.Interface{} 300 | 301 | err := netns.Do(func(hostNS ns.NetNS) error { 302 | hostVeth, contVeth0, err := ip.SetupVeth(ifName, mtu, hostNS) 303 | if err != nil { 304 | return err 305 | } 306 | hostInterface.Name = hostVeth.Name 307 | hostInterface.Mac = hostVeth.HardwareAddr.String() 308 | containerInterface.Name = contVeth0.Name 309 | // ip.SetupVeth does not retrieve MAC address from peer in veth 310 | containerNetlinkIface, _ := netlink.LinkByName(contVeth0.Name) 311 | containerInterface.Mac = containerNetlinkIface.Attrs().HardwareAddr.String() 312 | containerInterface.Sandbox = netns.Path() 313 | 314 | pr.Interfaces = append(pr.Interfaces, hostInterface, containerInterface) 315 | 316 | contVeth, err := net.InterfaceByName(ifName) 317 | if err != nil { 318 | return fmt.Errorf("failed to look up %q: %v", ifName, err) 319 | } 320 | 321 | if masq { 322 | // enable forwarding and SNATing for traffic rerouted from kube-proxy 323 | err := enableForwarding(containerIPV4, containerIPV6) 324 | if err != nil { 325 | return err 326 | } 327 | 328 | err = setupSNAT(k8sIfName, "kube-proxy SNAT") 329 | if err != nil { 330 | return fmt.Errorf("failed to enable SNAT on %q: %v", k8sIfName, err) 331 | } 332 | } 333 | 334 | // add host routes for each dst hostInterface ip on dev contVeth 335 | for _, ipc := range hostAddrs { 336 | addrBits := 128 337 | if ipc.IP.To4() != nil { 338 | addrBits = 32 339 | } 340 | 341 | err := netlink.RouteAdd(&netlink.Route{ 342 | LinkIndex: contVeth.Index, 343 | Scope: netlink.SCOPE_LINK, 344 | Dst: &net.IPNet{ 345 | IP: ipc.IP, 346 | Mask: net.CIDRMask(addrBits, addrBits), 347 | }, 348 | }) 349 | 350 | if err != nil { 351 | return fmt.Errorf("failed to add host route dst %v: %v", ipc.IP, err) 352 | } 353 | } 354 | 355 | // add a default gateway pointed at the first hostAddr 356 | err = netlink.RouteAdd(&netlink.Route{ 357 | LinkIndex: contVeth.Index, 358 | Scope: netlink.SCOPE_UNIVERSE, 359 | Dst: nil, 360 | Gw: hostAddrs[0].IP, 361 | }) 362 | if err != nil { 363 | return fmt.Errorf("failed to add default route %v: %v", hostAddrs[0].IP, err) 364 | } 365 | 366 | // Send a gratuitous arp for all borrowed v4 addresses 367 | for _, ipc := range pr.IPs { 368 | if ipc.Version == "4" { 369 | _ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth) 370 | } 371 | } 372 | 373 | return nil 374 | }) 375 | if err != nil { 376 | return nil, nil, err 377 | } 378 | return hostInterface, containerInterface, nil 379 | } 380 | 381 | func setupHostVeth(vethName string, hostAddrs []netlink.Addr, masq bool, tableStart int, result *current.Result) error { 382 | // no IPs to route 383 | if len(result.IPs) == 0 { 384 | return nil 385 | } 386 | 387 | // lookup by name as interface ids might have changed 388 | veth, err := net.InterfaceByName(vethName) 389 | if err != nil { 390 | return fmt.Errorf("failed to lookup %q: %v", vethName, err) 391 | } 392 | 393 | // add destination routes to Pod IPs 394 | for _, ipc := range result.IPs { 395 | addrBits := 128 396 | if ipc.Address.IP.To4() != nil { 397 | addrBits = 32 398 | } 399 | 400 | err := netlink.RouteAdd(&netlink.Route{ 401 | LinkIndex: veth.Index, 402 | Scope: netlink.SCOPE_LINK, 403 | Dst: &net.IPNet{ 404 | IP: ipc.Address.IP, 405 | Mask: net.CIDRMask(addrBits, addrBits), 406 | }, 407 | }) 408 | 409 | if err != nil { 410 | return fmt.Errorf("failed to add host route dst %v: %v", ipc.Address.IP, err) 411 | } 412 | } 413 | 414 | // add policy rules for traffic coming in from Pods and destined for the VPC 415 | err = addPolicyRules(veth, result.IPs[0], result.Routes, tableStart) 416 | if err != nil { 417 | return fmt.Errorf("failed to add policy rules: %v", err) 418 | } 419 | 420 | // Send a gratuitous arp for all borrowed v4 addresses 421 | for _, ipc := range hostAddrs { 422 | if ipc.IP.To4() != nil { 423 | _ = arping.GratuitousArpOverIface(ipc.IP, *veth) 424 | } 425 | } 426 | 427 | return nil 428 | } 429 | 430 | // cmdAdd is called for ADD requests 431 | func cmdAdd(args *skel.CmdArgs) error { 432 | conf, err := parseConfig(args.StdinData) 433 | if err != nil { 434 | return err 435 | } 436 | 437 | if conf.PrevResult == nil { 438 | return fmt.Errorf("must be called as chained plugin") 439 | } 440 | 441 | // This is some sample code to generate the list of container-side IPs. 442 | // We're casting the prevResult to a 0.3.0 response, which can also include 443 | // host-side IPs (but doesn't when converted from a 0.2.0 response). 444 | containerIPs := make([]net.IP, 0, len(conf.PrevResult.IPs)) 445 | if conf.CNIVersion != "0.3.0" { 446 | for _, ip := range conf.PrevResult.IPs { 447 | containerIPs = append(containerIPs, ip.Address.IP) 448 | } 449 | } else { 450 | for _, ip := range conf.PrevResult.IPs { 451 | if ip.Interface == nil { 452 | continue 453 | } 454 | intIdx := *ip.Interface 455 | // Every IP is indexed in to the interfaces array, with "-1" standing 456 | // for an unknown interface (which we'll assume to be Container-side 457 | // Skip all IPs we know belong to an interface with the wrong name. 458 | if intIdx >= 0 && intIdx < len(conf.PrevResult.Interfaces) && conf.PrevResult.Interfaces[intIdx].Name != args.IfName { 459 | continue 460 | } 461 | containerIPs = append(containerIPs, ip.Address.IP) 462 | } 463 | } 464 | if len(containerIPs) == 0 { 465 | return fmt.Errorf("got no container IPs") 466 | } 467 | 468 | iface, err := netlink.LinkByName(conf.HostInterface) 469 | if err != nil { 470 | return fmt.Errorf("failed to lookup %q: %v", conf.HostInterface, err) 471 | } 472 | 473 | hostAddrs, err := netlink.AddrList(iface, netlink.FAMILY_ALL) 474 | if err != nil || len(hostAddrs) == 0 { 475 | return fmt.Errorf("failed to get host IP addresses for %q: %v", iface, err) 476 | } 477 | 478 | netns, err := ns.GetNS(args.Netns) 479 | if err != nil { 480 | return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) 481 | } 482 | defer netns.Close() 483 | 484 | containerIPV4 := false 485 | containerIPV6 := false 486 | for _, ipc := range containerIPs { 487 | if ipc.To4() != nil { 488 | containerIPV4 = true 489 | } else { 490 | containerIPV6 = true 491 | } 492 | } 493 | 494 | hostInterface, _, err := setupContainerVeth(netns, conf.ContainerInterface, conf.MTU, 495 | hostAddrs, conf.IPMasq, containerIPV4, containerIPV6, args.IfName, conf.PrevResult) 496 | if err != nil { 497 | return err 498 | } 499 | 500 | if err = setupHostVeth(hostInterface.Name, hostAddrs, conf.IPMasq, conf.TableStart, conf.PrevResult); err != nil { 501 | return err 502 | } 503 | 504 | if conf.IPMasq { 505 | err := enableForwarding(containerIPV4, containerIPV6) 506 | if err != nil { 507 | return err 508 | } 509 | 510 | chain := utils.FormatChainName(conf.Name, args.ContainerID) 511 | comment := utils.FormatComment(conf.Name, args.ContainerID) 512 | for _, ipc := range containerIPs { 513 | // always assuming ipv4 514 | if err = ip.SetupIPMasq(&net.IPNet{IP: ipc, Mask: net.CIDRMask(32, 32)}, chain, comment); err != nil { 515 | return err 516 | } 517 | } 518 | } 519 | 520 | if err = setupNodePortRule(conf.HostInterface, conf.NodePorts, conf.NodePortMark); err != nil { 521 | return err 522 | } 523 | 524 | // Pass through the result for the next plugin 525 | return types.PrintResult(conf.PrevResult, conf.CNIVersion) 526 | } 527 | 528 | // cmdCheck is called for CHECK requests 529 | func cmdCheck(args *skel.CmdArgs) error { 530 | // TODO: implement this 531 | return nil 532 | } 533 | 534 | // cmdDel is called for DELETE requests 535 | func cmdDel(args *skel.CmdArgs) error { 536 | conf, err := parseConfig(args.StdinData) 537 | if err != nil { 538 | return fmt.Errorf("couldn't parse config: %w", err) 539 | } 540 | 541 | if !conf.IPMasq { 542 | // we don't have to do anything if IPMasq is false. 543 | return nil 544 | } 545 | 546 | if args.Netns == "" { 547 | return nil 548 | } 549 | 550 | // There is a netns so try to clean up. Delete can be called multiple times 551 | // so don't return an error if the device is already removed. 552 | // If the device isn't there then don't try to clean up IP masq either. 553 | var ( 554 | addrs []netlink.Addr 555 | vethPeerIndex = -1 556 | ) 557 | err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { 558 | var err error 559 | // use the container interface (veth0) to find the peer index, 560 | // so we can find this link outside of the namespace. 561 | _, vethPeerIndex, err = ip.GetVethPeerIfindex(conf.ContainerInterface) 562 | if err != nil { 563 | return fmt.Errorf("failed to lookup %q: %v", conf.ContainerInterface, err) 564 | } 565 | 566 | // now we grab the iface to get the proper container addrs 567 | iface, err := netlink.LinkByName(args.IfName) 568 | if err != nil { 569 | return fmt.Errorf("couldn't load link by name %s: %w", args.IfName, err) 570 | } 571 | // only care about errors in ipv4 space 572 | addrs, err = netlink.AddrList(iface, netlink.FAMILY_V4) 573 | if err != nil || len(addrs) == 0 { 574 | return fmt.Errorf("couldn't discover addrs from iface: %s: %w", args.IfName, err) 575 | } 576 | return err 577 | }) 578 | if err != nil { 579 | return fmt.Errorf("couldn't discover peer idx from netns %s: %w", args.Netns, err) 580 | } 581 | 582 | chain := utils.FormatChainName(conf.Name, args.ContainerID) 583 | comment := utils.FormatComment(conf.Name, args.ContainerID) 584 | for _, ipn := range addrs { 585 | // always assume ipv4, since those are the IPs we filter for 586 | if err := ip.TeardownIPMasq(&net.IPNet{IP: ipn.IP, Mask: net.CIDRMask(32, 32)}, chain, comment); err != nil { 587 | return fmt.Errorf("couldn't teardown ip masq: %w", err) 588 | } 589 | } 590 | 591 | if vethPeerIndex != -1 { 592 | link, err := netlink.LinkByIndex(vethPeerIndex) 593 | if err != nil { 594 | return fmt.Errorf("couldn't find link by index %d: %w", vethPeerIndex, err) 595 | } 596 | rule := netlink.NewRule() 597 | rule.IifName = link.Attrs().Name 598 | 599 | if err := netlink.RuleDel(rule); err != nil { 600 | return fmt.Errorf("couldn't delete rule %s: %w", rule.IifName, err) 601 | } 602 | if err := netlink.LinkDel(link); err != nil { 603 | return fmt.Errorf("couldn't delete link %s: %w", link.Attrs().Name, err) 604 | } 605 | } 606 | 607 | return nil 608 | } 609 | 610 | func main() { 611 | rand.Seed(time.Now().UnixNano()) 612 | skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, "unnumbered-ptp") 613 | } 614 | --------------------------------------------------------------------------------