├── MAINTAINERS ├── .gitignore ├── hack ├── test-go.sh ├── verify-all.sh ├── update-gofmt.sh ├── format.sh ├── lib │ ├── logging.sh │ ├── lib.sh │ └── version.sh ├── verify-gofmt.sh ├── build.sh └── build-img.sh ├── test ├── gpu_quota.json └── test_deploy.yaml ├── Makefile ├── go.mod ├── deploy └── gpu-admission.yaml ├── pkg ├── predicate │ ├── predicate.go │ ├── gpu_predicate_test.go │ └── gpu_predicate.go ├── version │ ├── base.go │ ├── version.go │ └── verflag │ │ └── verflag.go ├── device │ ├── deviceInfo.go │ ├── sort.go │ └── nodeInfo.go ├── algorithm │ ├── share.go │ ├── exclusive.go │ └── allocate.go ├── route │ └── routes.go └── util │ └── util.go ├── Dockerfile ├── README.md ├── main.go └── go.sum /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Thomas Song @mYmNeo 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .version-defs 2 | bin/**/* 3 | go/**/* 4 | .idea/ 5 | _output/ 6 | -------------------------------------------------------------------------------- /hack/test-go.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | TIMEOUT=${TIMEOUT:-5m} 8 | 9 | go test -v -test.timeout=${TIMEOUT} ./... 10 | -------------------------------------------------------------------------------- /hack/verify-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) 8 | 9 | # verify gofmt 10 | ${ROOT}/hack/verify-gofmt.sh 11 | -------------------------------------------------------------------------------- /hack/update-gofmt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) 8 | source $ROOT/hack/lib/lib.sh 9 | 10 | echo $(find_files) | xargs -n 1 -I {} goimports -w {} 11 | -------------------------------------------------------------------------------- /test/gpu_quota.json: -------------------------------------------------------------------------------- 1 | { 2 | "default": { 3 | "pool": [ "public" ], 4 | "quota": { 5 | "M40": 4, 6 | "P100": 4 7 | } 8 | }, 9 | "B": { 10 | "pool": [ "wx" ], 11 | "quota": { 12 | "M40": 8, 13 | "P100": 2 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /hack/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o nounset 4 | set -o pipefail 5 | 6 | ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) 7 | source "${ROOT}/hack/lib/lib.sh" 8 | 9 | GOIMPORT="gofmt -s -d -w" 10 | find_files | xargs $GOIMPORT 11 | if [ $? -ne 0 ]; then 12 | echo "Failed to format" 13 | else 14 | echo "Format successfully" 15 | fi 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .PHONY: all 3 | all: 4 | hack/build.sh 5 | 6 | .PHONY: clean 7 | clean: 8 | rm -rf bin/ _output/ go .version-defs 9 | 10 | .PHONY: build 11 | build: 12 | hack/build.sh 13 | 14 | # Run test 15 | .PHONY: test 16 | test: 17 | hack/test-go.sh 18 | 19 | .PHONY: verify 20 | verify: 21 | hack/verify-all.sh 22 | 23 | .PHONY: img 24 | img: 25 | hack/build-img.sh 26 | 27 | format: 28 | hack/format.sh 29 | 30 | # vim: set ts=2 sw=2 tw=0 noet : 31 | -------------------------------------------------------------------------------- /hack/lib/logging.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | api::log::usage() { 4 | echo >&2 5 | local message 6 | for message; do 7 | echo "$message" >&2 8 | done 9 | echo >&2 10 | } 11 | 12 | # Print a status line. Formatted to show up in a stream of output. 13 | api::log::status() { 14 | timestamp=$(date +"[%m%d %H:%M:%S]") 15 | echo "+++ $timestamp $1" 16 | shift 17 | for message; do 18 | echo " $message" 19 | done 20 | } 21 | # vim: set ts=2 sw=2 tw=0 et : 22 | -------------------------------------------------------------------------------- /test/test_deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: nginx-deployment 5 | spec: 6 | replicas: 2 7 | template: 8 | metadata: 9 | labels: 10 | app: nginx 11 | spec: 12 | containers: 13 | - name: nginx 14 | image: nginx:1.7.9 15 | resources: 16 | requests: 17 | memory: "4Mi" 18 | cpu: 1 19 | limits: 20 | alpha.kubernetes.io/nvidia-gpu: 2 21 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module tkestack.io/gpu-admission 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/gophercloud/gophercloud v0.1.0 // indirect 7 | github.com/julienschmidt/httprouter v1.3.1-0.20191005171706-08a3b3d20bbe 8 | github.com/spf13/pflag v1.0.5 9 | k8s.io/api v0.18.12 10 | k8s.io/apimachinery v0.18.12 11 | k8s.io/client-go v0.18.12 12 | k8s.io/component-base v0.18.12 13 | k8s.io/klog v1.0.0 14 | k8s.io/kube-scheduler v0.18.12 15 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /hack/lib/lib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | find_files() { 4 | find . -not \( \ 5 | \( \ 6 | -wholename './output' \ 7 | -o -wholename './_output' \ 8 | -o -wholename './_gopath' \ 9 | -o -wholename './release' \ 10 | -o -wholename './target' \ 11 | -o -wholename '*/third_party/*' \ 12 | -o -wholename '*/vendor/*' \ 13 | -o -wholename './staging' \ 14 | -o -wholename './pkg/client' \ 15 | \) -prune \ 16 | \) -name '*.go' 17 | } 18 | -------------------------------------------------------------------------------- /hack/verify-gofmt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) 8 | source $ROOT/hack/lib/lib.sh 9 | 10 | GOFMT="gofmt -s" 11 | bad_files=$(find_files | xargs $GOFMT -l) 12 | if [[ -n "${bad_files}" ]]; then 13 | echo "!!! '$GOFMT -w' needs to be run on the following files: " 14 | echo "${bad_files}" 15 | echo "run 'git ls-files -m | grep .go | xargs -n1 gofmt -s -w' to format your own code" 16 | exit 1 17 | fi 18 | -------------------------------------------------------------------------------- /deploy/gpu-admission.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | annotations: 5 | scheduler.alpha.kubernetes.io/critical-pod: "" 6 | name: gpu-admission 7 | namespace: kube-system 8 | spec: 9 | containers: 10 | - image: gpu-admission-tke:36 11 | imagePullPolicy: Always 12 | name: gpu-admission 13 | env: 14 | - name: LOG_LEVEL 15 | value: "4" 16 | ports: 17 | - containerPort: 3456 18 | dnsPolicy: ClusterFirstWithHostNet 19 | hostNetwork: true 20 | priority: 2000000000 21 | priorityClassName: system-cluster-critical -------------------------------------------------------------------------------- /hack/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) 8 | GIT_VERSION_FILE="${ROOT}/.version-defs" 9 | 10 | source "${ROOT}/hack/lib/version.sh" 11 | 12 | if [[ -f ${GIT_VERSION_FILE} ]]; then 13 | api::version::load_version_vars "${GIT_VERSION_FILE}" 14 | else 15 | api::version::get_version_vars 16 | api::version::save_version_vars "${GIT_VERSION_FILE}" 17 | fi 18 | 19 | go build -o bin/gpu-admission -ldflags "$(api::version::ldflags)" . 20 | if [ $? -eq 0 ]; then 21 | echo "Build success!" 22 | else 23 | echo "Faild to build!" 24 | fi 25 | -------------------------------------------------------------------------------- /pkg/predicate/predicate.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package predicate 18 | 19 | import ( 20 | extenderv1 "k8s.io/kube-scheduler/extender/v1" 21 | ) 22 | 23 | type Predicate interface { 24 | // Name returns the name of this predictor 25 | Name() string 26 | // Filter returns the filter result of predictor, this will tell the suitable nodes to running 27 | // pod 28 | Filter(args extenderv1.ExtenderArgs) *extenderv1.ExtenderFilterResult 29 | } 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 as build 2 | 3 | ARG version 4 | ARG commit 5 | 6 | RUN yum install -y rpm-build make 7 | 8 | ENV GOLANG_VERSION 1.13.4 9 | RUN curl -sSL https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz \ 10 | | tar -C /usr/local -xz 11 | ENV GOPATH /go 12 | ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH 13 | 14 | RUN mkdir -p /root/rpmbuild/{SPECS,SOURCES} 15 | 16 | COPY gpu-admission.spec /root/rpmbuild/SPECS 17 | COPY gpu-admission-source.tar.gz /root/rpmbuild/SOURCES 18 | 19 | RUN echo '%_topdir /root/rpmbuild' > /root/.rpmmacros \ 20 | && echo '%__os_install_post %{nil}' >> /root/.rpmmacros \ 21 | && echo '%debug_package %{nil}' >> /root/.rpmmacros 22 | WORKDIR /root/rpmbuild/SPECS 23 | RUN rpmbuild -ba --quiet \ 24 | --define 'version '${version}'' \ 25 | --define 'commit '${commit}'' \ 26 | gpu-admission.spec 27 | 28 | 29 | FROM centos:7 30 | 31 | ARG version 32 | ARG commit 33 | 34 | COPY --from=build /root/rpmbuild/RPMS/x86_64/gpu-admission-${version}-${commit}.el7.x86_64.rpm /tmp 35 | 36 | RUN rpm -ivh /tmp/gpu-admission-${version}-${commit}.el7.x86_64.rpm 37 | 38 | EXPOSE 3456 39 | 40 | CMD ["/bin/bash", "-c", "/usr/bin/gpu-admission --address=0.0.0.0:3456 --v=$LOG_LEVEL --logtostderr=true $EXTRA_FLAGS"] 41 | -------------------------------------------------------------------------------- /pkg/version/base.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package version 18 | 19 | // Base version information. 20 | // 21 | // This is the fallback data used when version information from git is not 22 | // provided via go ldflags. It provides an approximation of the Apiswitch 23 | // version for ad-hoc builds (e.g. `go build`) that cannot get the version 24 | // information from git. 25 | // 26 | // If you are looking at these fields in the git tree, they look 27 | // strange. They are modified on the fly by the build process. The 28 | // in-tree values are dummy values used for "git archive", which also 29 | // works for GitHub tar downloads. 30 | var ( 31 | // rpm package name, if install by rpm 32 | rpmPKGName string = "Not build from rpm package" 33 | // branch of git 34 | gitBranch string = "Not a git repo" 35 | // sha1 from git, output of $(git rev-parse HEAD) 36 | gitCommit string = "$Format:%H$" 37 | // state of git tree, either "clean" or "dirty" 38 | gitTreeState string = "Not a git tree" 39 | // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 40 | buildDate string = "1970-01-01T00:00:00Z" 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package version 18 | 19 | import ( 20 | "fmt" 21 | "runtime" 22 | ) 23 | 24 | // Info contains versioning information. 25 | // TODO: Add []string of api versions supported? It's still unclear 26 | // how we'll want to distribute that information. 27 | type Info struct { 28 | RpmPKGName string `json:"rpmPKGName"` 29 | GitBranch string `json:"gitBranch"` 30 | GitCommit string `json:"gitCommit"` 31 | GitTreeState string `json:"gitTreeState"` 32 | BuildDate string `json:"buildDate"` 33 | GoVersion string `json:"goVersion"` 34 | Compiler string `json:"compiler"` 35 | Platform string `json:"platform"` 36 | } 37 | 38 | // String returns info as a human-friendly version string. 39 | func (info Info) String() string { 40 | return info.GitBranch + "-" + info.GitCommit 41 | } 42 | 43 | // Get returns the overall codebase version. It's for detecting 44 | // what code a binary was built from. 45 | func Get() Info { 46 | // These variables typically come from -ldflags settings and in 47 | // their absence fallback to the settings in pkg/version/base.go 48 | return Info{ 49 | RpmPKGName: rpmPKGName, 50 | GitBranch: gitBranch, 51 | GitCommit: gitCommit, 52 | GitTreeState: gitTreeState, 53 | BuildDate: buildDate, 54 | GoVersion: runtime.Version(), 55 | Compiler: runtime.Compiler, 56 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pkg/device/deviceInfo.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package device 18 | 19 | import ( 20 | "fmt" 21 | 22 | "tkestack.io/gpu-admission/pkg/util" 23 | ) 24 | 25 | type DeviceInfo struct { 26 | id int 27 | totalMemory uint 28 | usedMemory uint 29 | usedCore uint 30 | } 31 | 32 | func newDeviceInfo(id int, totalMemory uint) *DeviceInfo { 33 | return &DeviceInfo{ 34 | id: id, 35 | totalMemory: totalMemory, 36 | } 37 | } 38 | 39 | // GetID returns the idx of this device 40 | func (dev *DeviceInfo) GetID() int { 41 | return dev.id 42 | } 43 | 44 | // AddUsedResources records the used GPU core and memory 45 | func (dev *DeviceInfo) AddUsedResources(usedCore uint, usedMemory uint) error { 46 | if usedCore+dev.usedCore > util.HundredCore { 47 | return fmt.Errorf("update usedcore failed, request: %d, already used: %d", 48 | usedCore, dev.usedCore) 49 | } 50 | 51 | if usedMemory+dev.usedMemory > dev.totalMemory { 52 | return fmt.Errorf("update usedmemory failed, request: %d, already used: %d", 53 | usedMemory, dev.usedMemory) 54 | } 55 | 56 | dev.usedCore += usedCore 57 | dev.usedMemory += usedMemory 58 | 59 | return nil 60 | } 61 | 62 | // AllocatableCores returns the remaining cores of this GPU device 63 | func (d *DeviceInfo) AllocatableCores() uint { 64 | return util.HundredCore - d.usedCore 65 | } 66 | 67 | // AllocatableMemory returns the remaining memory of this GPU device 68 | func (d *DeviceInfo) AllocatableMemory() uint { 69 | return d.totalMemory - d.usedMemory 70 | } 71 | -------------------------------------------------------------------------------- /pkg/device/sort.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package device 18 | 19 | //LessFunc represents funcion to compare two DeviceInfo or NodeInfo 20 | type LessFunc func(p1, p2 interface{}) bool 21 | 22 | var ( 23 | // ByAllocatableCores compares two device or node by allocatable cores 24 | ByAllocatableCores = func(p1, p2 interface{}) bool { 25 | var result bool 26 | switch p1.(type) { 27 | case *DeviceInfo: 28 | d1 := p1.(*DeviceInfo) 29 | d2 := p2.(*DeviceInfo) 30 | result = d1.AllocatableCores() < d2.AllocatableCores() 31 | case *NodeInfo: 32 | n1 := p1.(*NodeInfo) 33 | n2 := p2.(*NodeInfo) 34 | result = n1.GetAvailableCore() < n2.GetAvailableCore() 35 | } 36 | return result 37 | } 38 | 39 | // ByAllocatableMemory compares two device or node by allocatable memory 40 | ByAllocatableMemory = func(p1, p2 interface{}) bool { 41 | var result bool 42 | switch p1.(type) { 43 | case *DeviceInfo: 44 | d1 := p1.(*DeviceInfo) 45 | d2 := p2.(*DeviceInfo) 46 | result = d1.AllocatableMemory() < d2.AllocatableMemory() 47 | case *NodeInfo: 48 | n1 := p1.(*NodeInfo) 49 | n2 := p2.(*NodeInfo) 50 | result = n1.GetAvailableMemory() < n2.GetAvailableMemory() 51 | } 52 | return result 53 | } 54 | 55 | ByID = func(p1, p2 interface{}) bool { 56 | var result bool 57 | switch p1.(type) { 58 | case *DeviceInfo: 59 | d1 := p1.(*DeviceInfo) 60 | d2 := p2.(*DeviceInfo) 61 | result = d1.GetID() < d2.GetID() 62 | case *NodeInfo: 63 | n1 := p1.(*NodeInfo) 64 | n2 := p2.(*NodeInfo) 65 | result = n1.GetName() < n2.GetName() 66 | } 67 | return result 68 | } 69 | ) 70 | -------------------------------------------------------------------------------- /hack/build-img.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # RPM package deployment 4 | set -o errexit 5 | set -o nounset 6 | set -o pipefail 7 | 8 | GITCOMMIT=$(git log --oneline|wc -l|sed -e 's/^[ \t]*//') 9 | VERSION=${VERSION:-unknown} # MUST modify it for every branch! 10 | ROOT=$(dirname "${BASH_SOURCE}")/.. 11 | IMAGE=${IMAGE:-"gpu-admission-${VERSION}:${GITCOMMIT}"} 12 | GIT_VERSION_FILE="${ROOT}/.version-defs" 13 | 14 | readonly LOCAL_OUTPUT_ROOT="${ROOT}/${OUT_DIR:-_output}" 15 | 16 | source "${ROOT}/hack/lib/logging.sh" 17 | source "${ROOT}/hack/lib/version.sh" 18 | 19 | function api::build::ensure_tar() { 20 | if [[ -n "${TAR:-}" ]]; then 21 | return 22 | fi 23 | 24 | # Find gnu tar if it is available, bomb out if not. 25 | TAR=tar 26 | if which gtar &>/dev/null; then 27 | TAR=gtar 28 | else 29 | if which gnutar &>/dev/null; then 30 | TAR=gnutar 31 | fi 32 | fi 33 | if ! "${TAR}" --version | grep -q GNU; then 34 | echo " !!! Cannot find GNU tar. Build on Linux or install GNU tar" 35 | echo " on Mac OS X (brew install gnu-tar)." 36 | return 1 37 | fi 38 | } 39 | 40 | # The set of source targets to include in the api-build image 41 | function api::build::source_targets() { 42 | local targets=( 43 | $(find . -mindepth 1 -maxdepth 1 -not \( \ 44 | \( -path ./_\* -o -path ./.git\* -o -path ./_output -o -path ./bin -o -path ./go \) -prune \ 45 | \)) 46 | ) 47 | echo "${targets[@]}" 48 | } 49 | 50 | function api::build::prepare_build() { 51 | api::build::ensure_tar || return 1 52 | 53 | mkdir -p "${LOCAL_OUTPUT_ROOT}" 54 | "${TAR}" czf "${LOCAL_OUTPUT_ROOT}/gpu-admission-source.tar.gz" --transform 's,^,/gpu-admission-'$VERSION'/,' $(api::build::source_targets) 55 | 56 | cp -R "${ROOT}/build/gpu-admission.spec" "${LOCAL_OUTPUT_ROOT}" 57 | cp "${ROOT}/Dockerfile" "${LOCAL_OUTPUT_ROOT}" 58 | } 59 | 60 | function api::build::generate() { 61 | api::log::status "Generating image..." 62 | ( 63 | cd "${LOCAL_OUTPUT_ROOT}" 64 | docker build -t $IMAGE \ 65 | --build-arg version=${VERSION} \ 66 | --build-arg commit=${GITCOMMIT} \ 67 | . 68 | ) 69 | } 70 | 71 | if [[ -f ${GIT_VERSION_FILE} ]]; then 72 | api::version::load_version_vars "${GIT_VERSION_FILE}" 73 | else 74 | api::version::get_version_vars 75 | api::version::save_version_vars "${ROOT}/.version-defs" 76 | fi 77 | api::build::prepare_build 78 | api::build::generate 79 | 80 | # vim: set ts=2 sw=2 tw=0 et : 81 | -------------------------------------------------------------------------------- /pkg/version/verflag/verflag.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package verflag 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "strconv" 23 | 24 | "tkestack.io/gpu-admission/pkg/version" 25 | 26 | flag "github.com/spf13/pflag" 27 | ) 28 | 29 | type versionValue int 30 | 31 | const ( 32 | VersionFalse versionValue = 0 33 | VersionTrue versionValue = 1 34 | VersionRaw versionValue = 2 35 | ) 36 | 37 | const strRawVersion string = "raw" 38 | 39 | func (v *versionValue) IsBoolFlag() bool { 40 | return true 41 | } 42 | 43 | func (v *versionValue) Get() interface{} { 44 | return versionValue(*v) 45 | } 46 | 47 | func (v *versionValue) Set(s string) error { 48 | if s == strRawVersion { 49 | *v = VersionRaw 50 | return nil 51 | } 52 | boolVal, err := strconv.ParseBool(s) 53 | if boolVal { 54 | *v = VersionTrue 55 | } else { 56 | *v = VersionFalse 57 | } 58 | return err 59 | } 60 | 61 | func (v *versionValue) String() string { 62 | if *v == VersionRaw { 63 | return strRawVersion 64 | } 65 | return fmt.Sprintf("%v", bool(*v == VersionTrue)) 66 | } 67 | 68 | // The type of the flag as required by the pflag.Value interface 69 | func (v *versionValue) Type() string { 70 | return "version" 71 | } 72 | 73 | func VersionVar(p *versionValue, name string, value versionValue, usage string) { 74 | *p = value 75 | flag.Var(p, name, usage) 76 | // "--version" will be treated as "--version=true" 77 | flag.Lookup(name).NoOptDefVal = "true" 78 | } 79 | 80 | func Version(name string, value versionValue, usage string) *versionValue { 81 | p := new(versionValue) 82 | VersionVar(p, name, value, usage) 83 | return p 84 | } 85 | 86 | var ( 87 | versionFlag = Version("version", VersionFalse, "Print version information and quit") 88 | ) 89 | 90 | // PrintAndExitIfRequested will check if the -version flag was passed 91 | // and, if so, print the version and exit. 92 | func PrintAndExitIfRequested() { 93 | if *versionFlag == VersionRaw { 94 | fmt.Printf("%#v\n", version.Get()) 95 | os.Exit(0) 96 | } else if *versionFlag == VersionTrue { 97 | fmt.Printf("GPU-Quota-Admission %s\n", version.Get()) 98 | os.Exit(0) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GPU admission 2 | 3 | It is a [scheduler extender](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/scheduler_extender.md) for GPU admission. 4 | It provides the following features: 5 | 6 | - provides quota limitation according to GPU device type 7 | - avoids fragment allocation of node by working with [gpu-manager](https://github.com/tkestack/gpu-manager) 8 | 9 | > For more details, please refer to the documents in `docs` directory in this project 10 | 11 | 12 | ## 1. Build 13 | 14 | ``` 15 | $ make build 16 | ``` 17 | 18 | ## 2. Run 19 | 20 | ### 2.1 Run gpu-admission. 21 | 22 | ``` 23 | $ bin/gpu-admission --address=127.0.0.1:3456 --v=4 --kubeconfig --logtostderr=true 24 | ``` 25 | 26 | Other options 27 | 28 | ``` 29 | --address string The address it will listen (default "127.0.0.1:3456") 30 | --alsologtostderr log to standard error as well as files 31 | --kubeconfig string Path to a kubeconfig. Only required if out-of-cluster. 32 | --log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0) 33 | --log-dir string If non-empty, write log files in this directory 34 | --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) 35 | --logtostderr log to standard error instead of files (default true) 36 | --master string The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster. 37 | --pprofAddress string The address for debug (default "127.0.0.1:3457") 38 | --stderrthreshold severity logs at or above this threshold go to stderr (default 2) 39 | -v, --v Level number for the log level verbosity 40 | --version version[=true] Print version information and quit 41 | --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging 42 | ``` 43 | 44 | ### 2.2 Configure kube-scheduler policy file, and run a kubernetes cluster. 45 | 46 | Example for scheduler-policy-config.json: 47 | ``` 48 | { 49 | "kind": "Policy", 50 | "apiVersion": "v1", 51 | "predicates": [ 52 | { 53 | "name": "PodFitsHostPorts" 54 | }, 55 | { 56 | "name": "PodFitsResources" 57 | }, 58 | { 59 | "name": "NoDiskConflict" 60 | }, 61 | { 62 | "name": "MatchNodeSelector" 63 | }, 64 | { 65 | "name": "HostName" 66 | } 67 | ], 68 | "extenders": [ 69 | { 70 | "urlPrefix": "http://:/scheduler", 71 | "apiVersion": "v1beta1", 72 | "filterVerb": "predicates", 73 | "enableHttps": false, 74 | "nodeCacheCapable": false 75 | } 76 | ], 77 | "hardPodAffinitySymmetricWeight": 10, 78 | "alwaysCheckAllPredicates": false 79 | } 80 | ``` 81 | 82 | Do not forget to add config for scheduler: `--policy-config-file=XXX --use-legacy-policy-config=true`. 83 | Keep this extender as the last one of all scheduler extenders. 84 | -------------------------------------------------------------------------------- /pkg/algorithm/share.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package algorithm 18 | 19 | import ( 20 | "sort" 21 | 22 | "k8s.io/klog" 23 | 24 | "tkestack.io/gpu-admission/pkg/device" 25 | ) 26 | 27 | type shareMode struct { 28 | node *device.NodeInfo 29 | } 30 | 31 | //NewShareMode returns a new shareMode struct. 32 | // 33 | //Evaluate() of shareMode returns one device with minimum available cores 34 | //which fullfil the request. 35 | // 36 | //Share mode means multiple application may share one GPU device which uses 37 | //GPU more efficiently. 38 | func NewShareMode(n *device.NodeInfo) *shareMode { 39 | return &shareMode{n} 40 | } 41 | 42 | func (al *shareMode) Evaluate(cores uint, memory uint) []*device.DeviceInfo { 43 | var ( 44 | devs []*device.DeviceInfo 45 | deviceCount = al.node.GetDeviceCount() 46 | tmpStore = make([]*device.DeviceInfo, deviceCount) 47 | sorter = shareModeSort(device.ByAllocatableCores, device.ByAllocatableMemory, device.ByID) 48 | ) 49 | 50 | for i := 0; i < deviceCount; i++ { 51 | tmpStore[i] = al.node.GetDeviceMap()[i] 52 | } 53 | 54 | sorter.Sort(tmpStore) 55 | 56 | for _, dev := range tmpStore { 57 | if dev.AllocatableCores() >= cores && dev.AllocatableMemory() >= memory { 58 | klog.V(4).Infof("Pick up %d , cores: %d, memory: %d", 59 | dev.GetID(), dev.AllocatableCores(), dev.AllocatableMemory()) 60 | devs = append(devs, dev) 61 | break 62 | } 63 | } 64 | 65 | return devs 66 | } 67 | 68 | type shareModePriority struct { 69 | data []*device.DeviceInfo 70 | less []device.LessFunc 71 | } 72 | 73 | func shareModeSort(less ...device.LessFunc) *shareModePriority { 74 | return &shareModePriority{ 75 | less: less, 76 | } 77 | } 78 | 79 | func (smp *shareModePriority) Sort(data []*device.DeviceInfo) { 80 | smp.data = data 81 | sort.Sort(smp) 82 | } 83 | 84 | func (smp *shareModePriority) Len() int { 85 | return len(smp.data) 86 | } 87 | 88 | func (smp *shareModePriority) Swap(i, j int) { 89 | smp.data[i], smp.data[j] = smp.data[j], smp.data[i] 90 | } 91 | 92 | func (smp *shareModePriority) Less(i, j int) bool { 93 | var k int 94 | 95 | for k = 0; k < len(smp.less)-1; k++ { 96 | less := smp.less[k] 97 | switch { 98 | case less(smp.data[i], smp.data[j]): 99 | return true 100 | case less(smp.data[j], smp.data[i]): 101 | return false 102 | } 103 | } 104 | 105 | return smp.less[k](smp.data[i], smp.data[j]) 106 | } 107 | -------------------------------------------------------------------------------- /pkg/algorithm/exclusive.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package algorithm 18 | 19 | import ( 20 | "sort" 21 | 22 | "k8s.io/klog" 23 | 24 | "tkestack.io/gpu-admission/pkg/device" 25 | "tkestack.io/gpu-admission/pkg/util" 26 | ) 27 | 28 | type exclusiveMode struct { 29 | node *device.NodeInfo 30 | } 31 | 32 | //NewExclusiveMode returns a new exclusiveMode struct. 33 | // 34 | //Evaluate() of exclusiveMode returns one or more empty devices 35 | //which fullfil the request. 36 | // 37 | //Exclusive mode means GPU devices are not sharing, only one 38 | //application can use them. 39 | func NewExclusiveMode(n *device.NodeInfo) *exclusiveMode { 40 | return &exclusiveMode{n} 41 | } 42 | 43 | func (al *exclusiveMode) Evaluate(cores uint, _ uint) []*device.DeviceInfo { 44 | var ( 45 | devs []*device.DeviceInfo 46 | deviceCount = al.node.GetDeviceCount() 47 | tmpStore = make([]*device.DeviceInfo, deviceCount) 48 | sorter = exclusiveModeSort( 49 | device.ByAllocatableCores, 50 | device.ByAllocatableMemory, 51 | device.ByID) 52 | num = int(cores / util.HundredCore) 53 | ) 54 | 55 | for i := 0; i < deviceCount; i++ { 56 | tmpStore[i] = al.node.GetDeviceMap()[i] 57 | } 58 | 59 | sorter.Sort(tmpStore) 60 | 61 | for _, dev := range tmpStore { 62 | if num == 0 { 63 | break 64 | } 65 | if dev.AllocatableCores() == util.HundredCore { 66 | devs = append(devs, dev) 67 | num -= 1 68 | continue 69 | } 70 | } 71 | 72 | if num > 0 { 73 | return nil 74 | } 75 | 76 | if klog.V(2) { 77 | for _, dev := range devs { 78 | klog.V(4).Infof("Pick up %d , cores: %d, memory: %d", 79 | dev.GetID(), dev.AllocatableCores(), dev.AllocatableMemory()) 80 | } 81 | } 82 | 83 | return devs 84 | } 85 | 86 | type exclusiveModePriority struct { 87 | data []*device.DeviceInfo 88 | less []device.LessFunc 89 | } 90 | 91 | func exclusiveModeSort(less ...device.LessFunc) *exclusiveModePriority { 92 | return &exclusiveModePriority{ 93 | less: less, 94 | } 95 | } 96 | 97 | func (emp *exclusiveModePriority) Sort(data []*device.DeviceInfo) { 98 | emp.data = data 99 | sort.Sort(emp) 100 | } 101 | 102 | func (emp *exclusiveModePriority) Len() int { 103 | return len(emp.data) 104 | } 105 | 106 | func (emp *exclusiveModePriority) Swap(i, j int) { 107 | emp.data[i], emp.data[j] = emp.data[j], emp.data[i] 108 | } 109 | 110 | func (emp *exclusiveModePriority) Less(i, j int) bool { 111 | var k int 112 | 113 | for k = 0; k < len(emp.less)-1; k++ { 114 | less := emp.less[k] 115 | switch { 116 | case less(emp.data[i], emp.data[j]): 117 | return true 118 | case less(emp.data[j], emp.data[i]): 119 | return false 120 | } 121 | } 122 | 123 | return emp.less[k](emp.data[i], emp.data[j]) 124 | } 125 | -------------------------------------------------------------------------------- /pkg/route/routes.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package route 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "fmt" 23 | "io" 24 | "net/http" 25 | 26 | "github.com/julienschmidt/httprouter" 27 | "k8s.io/klog" 28 | extenderv1 "k8s.io/kube-scheduler/extender/v1" 29 | 30 | "tkestack.io/gpu-admission/pkg/predicate" 31 | "tkestack.io/gpu-admission/pkg/version" 32 | ) 33 | 34 | const ( 35 | // version router path 36 | versionPath = "/version" 37 | apiPrefix = "/scheduler" 38 | // predication router path 39 | predicatesPrefix = apiPrefix + "/predicates" 40 | ) 41 | 42 | func checkBody(w http.ResponseWriter, r *http.Request) { 43 | if r.Body == nil { 44 | http.Error(w, "Please send a request body", 400) 45 | return 46 | } 47 | } 48 | 49 | // PredicateRoute sets router table for predication 50 | func PredicateRoute(predicate predicate.Predicate) httprouter.Handle { 51 | return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 52 | checkBody(w, r) 53 | 54 | var buf bytes.Buffer 55 | body := io.TeeReader(r.Body, &buf) 56 | 57 | var extenderArgs extenderv1.ExtenderArgs 58 | var extenderFilterResult *extenderv1.ExtenderFilterResult 59 | 60 | if err := json.NewDecoder(body).Decode(&extenderArgs); err != nil { 61 | extenderFilterResult = &extenderv1.ExtenderFilterResult{ 62 | Nodes: nil, 63 | FailedNodes: nil, 64 | Error: err.Error(), 65 | } 66 | } else { 67 | extenderFilterResult = predicate.Filter(extenderArgs) 68 | klog.V(4).Infof("%s: ExtenderArgs = %+v", predicate.Name(), extenderArgs) 69 | } 70 | 71 | if resultBody, err := json.Marshal(extenderFilterResult); err != nil { 72 | klog.Errorf("Failed to marshal extenderFilterResult: %+v, %+v", 73 | err, extenderFilterResult) 74 | w.Header().Set("Content-Type", "application/json") 75 | w.WriteHeader(http.StatusInternalServerError) 76 | w.Write([]byte(err.Error())) 77 | } else { 78 | klog.V(4).Infof("%s: extenderFilterResult = %s", 79 | predicate.Name(), string(resultBody)) 80 | w.Header().Set("Content-Type", "application/json") 81 | w.WriteHeader(http.StatusOK) 82 | w.Write(resultBody) 83 | } 84 | } 85 | } 86 | 87 | // VersionRoute returns the version of router in response 88 | func VersionRoute(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 89 | fmt.Fprint(w, fmt.Sprint(version.Get())) 90 | } 91 | 92 | func AddVersion(router *httprouter.Router) { 93 | router.GET(versionPath, DebugLogging(VersionRoute, versionPath)) 94 | } 95 | 96 | // DebugLogging wraps handler for debugging purposes 97 | func DebugLogging(h httprouter.Handle, path string) httprouter.Handle { 98 | return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { 99 | klog.V(10).Infof("%s request body = %s", path, r.Body) 100 | h(w, r, p) 101 | klog.V(10).Infof("%s response = %s", path, w) 102 | } 103 | } 104 | 105 | func AddPredicate(router *httprouter.Router, predicate predicate.Predicate) { 106 | path := predicatesPrefix 107 | router.POST(path, DebugLogging(PredicateRoute(predicate), path)) 108 | } 109 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "log" 22 | "net/http" 23 | _ "net/http/pprof" 24 | "strings" 25 | 26 | "github.com/julienschmidt/httprouter" 27 | "github.com/spf13/pflag" 28 | "k8s.io/client-go/kubernetes" 29 | "k8s.io/client-go/rest" 30 | "k8s.io/client-go/tools/clientcmd" 31 | "k8s.io/component-base/logs" 32 | "k8s.io/klog" 33 | 34 | "tkestack.io/gpu-admission/pkg/predicate" 35 | "tkestack.io/gpu-admission/pkg/route" 36 | "tkestack.io/gpu-admission/pkg/version/verflag" 37 | ) 38 | 39 | var ( 40 | kubeconfig string 41 | masterURL string 42 | listenAddress string 43 | profileAddress string 44 | ) 45 | 46 | func main() { 47 | addFlags(pflag.CommandLine) 48 | 49 | logs.InitLogs() 50 | defer logs.FlushLogs() 51 | initFlags() 52 | flag.CommandLine.Parse([]string{}) 53 | verflag.PrintAndExitIfRequested() 54 | 55 | router := httprouter.New() 56 | route.AddVersion(router) 57 | 58 | var ( 59 | clientCfg *rest.Config 60 | err error 61 | ) 62 | 63 | clientCfg, err = clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) 64 | if err != nil { 65 | klog.Fatalf("Error building kubeconfig: %s", err.Error()) 66 | } 67 | 68 | kubeClient, err := kubernetes.NewForConfig(clientCfg) 69 | if err != nil { 70 | klog.Fatalf("Error building kubernetes clientset: %s", err.Error()) 71 | } 72 | 73 | gpuFilter, err := predicate.NewGPUFilter(kubeClient) 74 | if err != nil { 75 | klog.Fatalf("Failed to new gpu quota filter: %s", err.Error()) 76 | } 77 | route.AddPredicate(router, gpuFilter) 78 | 79 | go func() { 80 | log.Println(http.ListenAndServe(profileAddress, nil)) 81 | }() 82 | 83 | klog.Infof("Server starting on %s", listenAddress) 84 | if err := http.ListenAndServe(listenAddress, router); err != nil { 85 | log.Fatal(err) 86 | } 87 | } 88 | 89 | func addFlags(fs *pflag.FlagSet) { 90 | fs.StringVar(&kubeconfig, "kubeconfig", "", 91 | "Path to a kubeconfig. Only required if out-of-cluster.") 92 | fs.StringVar(&masterURL, "master", "", 93 | "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") 94 | fs.StringVar(&listenAddress, "address", "127.0.0.1:3456", "The address it will listen") 95 | fs.StringVar(&profileAddress, "pprofAddress", "127.0.0.1:3457", "The address for debug") 96 | } 97 | 98 | func wordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { 99 | if strings.Contains(name, "_") { 100 | return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) 101 | } 102 | return pflag.NormalizedName(name) 103 | } 104 | 105 | // InitFlags normalizes and parses the command line flags 106 | func initFlags() { 107 | pflag.CommandLine.SetNormalizeFunc(wordSepNormalizeFunc) 108 | // Only glog flags will be added 109 | flag.CommandLine.VisitAll(func(goflag *flag.Flag) { 110 | switch goflag.Name { 111 | case "logtostderr", "alsologtostderr", 112 | "v", "stderrthreshold", "vmodule", "log_backtrace_at", "log_dir": 113 | pflag.CommandLine.AddGoFlag(goflag) 114 | } 115 | }) 116 | 117 | pflag.Parse() 118 | } 119 | -------------------------------------------------------------------------------- /pkg/algorithm/allocate.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package algorithm 18 | 19 | import ( 20 | "fmt" 21 | "strconv" 22 | "strings" 23 | "time" 24 | 25 | "k8s.io/api/core/v1" 26 | "k8s.io/klog" 27 | 28 | "tkestack.io/gpu-admission/pkg/device" 29 | "tkestack.io/gpu-admission/pkg/util" 30 | ) 31 | 32 | type allocator struct { 33 | nodeInfo *device.NodeInfo 34 | } 35 | 36 | func NewAllocator(n *device.NodeInfo) *allocator { 37 | return &allocator{nodeInfo: n} 38 | } 39 | 40 | // IsAllocatable attempt to allocate containers which has GPU request of given pod 41 | func (alloc *allocator) IsAllocatable(pod *v1.Pod) bool { 42 | allocatable := true 43 | for _, c := range pod.Spec.Containers { 44 | if !util.IsGPURequiredContainer(&c) { 45 | continue 46 | } 47 | _, err := alloc.AllocateOne(&c) 48 | if err != nil { 49 | klog.Infof("failed to allocate for pod %s container %s", pod.UID, c.Name) 50 | allocatable = false 51 | break 52 | } 53 | } 54 | return allocatable 55 | } 56 | 57 | // Allocate tries to find a suitable GPU device for containers 58 | // and records some data in pod's annotation 59 | func (alloc *allocator) Allocate(pod *v1.Pod) (*v1.Pod, error) { 60 | newPod := pod.DeepCopy() 61 | if newPod.Annotations == nil { 62 | newPod.Annotations = make(map[string]string) 63 | } 64 | for i, c := range newPod.Spec.Containers { 65 | if !util.IsGPURequiredContainer(&c) { 66 | continue 67 | } 68 | devIDs := []string{} 69 | devs, err := alloc.AllocateOne(&c) 70 | if err != nil { 71 | klog.Infof("failed to allocate for pod %s(%s)", newPod.Name, c.Name) 72 | return nil, err 73 | } 74 | for _, dev := range devs { 75 | devIDs = append(devIDs, strconv.Itoa(dev.GetID())) 76 | } 77 | newPod.Annotations[util.PredicateGPUIndexPrefix+strconv.Itoa(i)] = strings.Join(devIDs, ",") 78 | } 79 | newPod.Annotations[util.PredicateNode] = alloc.nodeInfo.GetName() 80 | newPod.Annotations[util.GPUAssigned] = "false" 81 | newPod.Annotations[util.PredicateTimeAnnotation] = fmt.Sprintf("%d", time.Now().UnixNano()) 82 | 83 | return newPod, nil 84 | } 85 | 86 | // AllocateOne tries to allocate GPU devices for given container 87 | func (alloc *allocator) AllocateOne(container *v1.Container) ([]*device.DeviceInfo, error) { 88 | var ( 89 | devs []*device.DeviceInfo 90 | sharedMode bool 91 | vcore, vmemory uint 92 | ) 93 | node := alloc.nodeInfo.GetNode() 94 | nodeTotalMemory := util.GetCapacityOfNode(node, util.VMemoryAnnotation) 95 | deviceCount := util.GetGPUDeviceCountOfNode(node) 96 | deviceTotalMemory := uint(nodeTotalMemory / deviceCount) 97 | needCores := util.GetGPUResourceOfContainer(container, util.VCoreAnnotation) 98 | needMemory := util.GetGPUResourceOfContainer(container, util.VMemoryAnnotation) 99 | 100 | switch { 101 | case needCores < util.HundredCore: 102 | devs = NewShareMode(alloc.nodeInfo).Evaluate(needCores, needMemory) 103 | sharedMode = true 104 | default: 105 | devs = NewExclusiveMode(alloc.nodeInfo).Evaluate(needCores, needMemory) 106 | } 107 | 108 | if len(devs) == 0 { 109 | return nil, fmt.Errorf("failed to allocate for container %s", container.Name) 110 | } 111 | 112 | if sharedMode { 113 | vcore = needCores 114 | vmemory = needMemory 115 | } else { 116 | vcore = util.HundredCore 117 | vmemory = deviceTotalMemory 118 | } 119 | 120 | // record this container GPU request, we don't rollback data if an error happened, 121 | // because any container failed to be allocated will cause the predication failed 122 | for _, dev := range devs { 123 | err := alloc.nodeInfo.AddUsedResources(dev.GetID(), vcore, vmemory) 124 | if err != nil { 125 | klog.Infof("failed to update used resource for node %s dev %d due to %v", 126 | node.Name, dev.GetID(), err) 127 | return nil, err 128 | } 129 | } 130 | return devs, nil 131 | } 132 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package util 18 | 19 | import ( 20 | "fmt" 21 | "strconv" 22 | "strings" 23 | 24 | "k8s.io/api/core/v1" 25 | apierr "k8s.io/apimachinery/pkg/api/errors" 26 | "k8s.io/klog" 27 | ) 28 | 29 | const ( 30 | VCoreAnnotation = "tencent.com/vcuda-core" 31 | VMemoryAnnotation = "tencent.com/vcuda-memory" 32 | PredicateTimeAnnotation = "tencent.com/predicate-time" 33 | PredicateGPUIndexPrefix = "tencent.com/predicate-gpu-idx-" 34 | PredicateNode = "tencent.com/predicate-node" 35 | GPUAssigned = "tencent.com/gpu-assigned" 36 | HundredCore = 100 37 | ) 38 | 39 | // IsGPURequiredPod tell if the pod is a GPU request pod 40 | func IsGPURequiredPod(pod *v1.Pod) bool { 41 | klog.V(4).Infof("Determine if the pod %s needs GPU resource", pod.Name) 42 | 43 | vcore := GetGPUResourceOfPod(pod, VCoreAnnotation) 44 | vmemory := GetGPUResourceOfPod(pod, VMemoryAnnotation) 45 | 46 | // Check if pod request for GPU resource 47 | if vcore <= 0 || (vcore < HundredCore && vmemory <= 0) { 48 | klog.V(4).Infof("Pod %s in namespace %s does not Request for GPU resource", 49 | pod.Name, 50 | pod.Namespace) 51 | return false 52 | } 53 | 54 | return true 55 | } 56 | 57 | // IsGPURequiredContainer tell if the container is a GPU request container 58 | func IsGPURequiredContainer(c *v1.Container) bool { 59 | klog.V(4).Infof("Determine if the container %s needs GPU resource", c.Name) 60 | 61 | vcore := GetGPUResourceOfContainer(c, VCoreAnnotation) 62 | vmemory := GetGPUResourceOfContainer(c, VMemoryAnnotation) 63 | 64 | // Check if container request for GPU resource 65 | if vcore <= 0 || (vcore < HundredCore && vmemory <= 0) { 66 | klog.V(4).Infof("Container %s does not Request for GPU resource", c.Name) 67 | return false 68 | } 69 | 70 | return true 71 | } 72 | 73 | // GetGPUResourceOfPod returns the limit size of GPU resource of given pod 74 | func GetGPUResourceOfPod(pod *v1.Pod, resourceName v1.ResourceName) uint { 75 | var total uint 76 | containers := pod.Spec.Containers 77 | for _, container := range containers { 78 | if val, ok := container.Resources.Limits[resourceName]; ok { 79 | total += uint(val.Value()) 80 | } 81 | } 82 | return total 83 | } 84 | 85 | // GetGPUResourceOfPod returns the limit size of GPU resource of given container 86 | func GetGPUResourceOfContainer(container *v1.Container, resourceName v1.ResourceName) uint { 87 | var count uint 88 | if val, ok := container.Resources.Limits[resourceName]; ok { 89 | count = uint(val.Value()) 90 | } 91 | return count 92 | } 93 | 94 | // Is the Node has GPU device 95 | func IsGPUEnabledNode(node *v1.Node) bool { 96 | return GetCapacityOfNode(node, VCoreAnnotation) > 0 97 | } 98 | 99 | // Get the capacity of request resource of the Node 100 | func GetCapacityOfNode(node *v1.Node, resourceName string) int { 101 | val, ok := node.Status.Capacity[v1.ResourceName(resourceName)] 102 | 103 | if !ok { 104 | return 0 105 | } 106 | 107 | return int(val.Value()) 108 | } 109 | 110 | // GetGPUDeviceCountOfNode returns the number of GPU devices 111 | func GetGPUDeviceCountOfNode(node *v1.Node) int { 112 | val, ok := node.Status.Capacity[VCoreAnnotation] 113 | if !ok { 114 | return 0 115 | } 116 | return int(val.Value()) / HundredCore 117 | } 118 | 119 | // GetPredicateIdxOfContainer returns the idx number of given container should be run on which 120 | // GPU device 121 | func GetPredicateIdxOfContainer(pod *v1.Pod, containerIndex int) ([]int, error) { 122 | var ret []int 123 | predicateIndexes, ok := pod.Annotations[PredicateGPUIndexPrefix+strconv.Itoa(containerIndex)] 124 | if !ok { 125 | return ret, fmt.Errorf("predicate index for container %d of pod %s not found", 126 | containerIndex, pod.UID) 127 | } 128 | for _, indexStr := range strings.Split(predicateIndexes, ",") { 129 | index, err := strconv.Atoi(indexStr) 130 | if err != nil { 131 | return ret, err 132 | } 133 | ret = append(ret, index) 134 | } 135 | return ret, nil 136 | } 137 | 138 | func ShouldRetry(err error) bool { 139 | return apierr.IsConflict(err) || apierr.IsServerTimeout(err) 140 | } 141 | -------------------------------------------------------------------------------- /pkg/device/nodeInfo.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package device 18 | 19 | import ( 20 | "sort" 21 | 22 | "k8s.io/api/core/v1" 23 | "k8s.io/klog" 24 | 25 | "tkestack.io/gpu-admission/pkg/util" 26 | ) 27 | 28 | type NodeInfo struct { 29 | name string 30 | node *v1.Node 31 | devs map[int]*DeviceInfo 32 | deviceCount int 33 | totalMemory uint 34 | usedCore uint 35 | usedMemory uint 36 | } 37 | 38 | func NewNodeInfo(node *v1.Node, pods []*v1.Pod) *NodeInfo { 39 | klog.V(4).Infof("debug: NewNodeInfo() creates nodeInfo for %s", node.Name) 40 | 41 | devMap := map[int]*DeviceInfo{} 42 | nodeTotalMemory := uint(util.GetCapacityOfNode(node, util.VMemoryAnnotation)) 43 | deviceCount := util.GetGPUDeviceCountOfNode(node) 44 | deviceTotalMemory := nodeTotalMemory / uint(deviceCount) 45 | for i := 0; i < deviceCount; i++ { 46 | devMap[i] = newDeviceInfo(i, deviceTotalMemory) 47 | } 48 | 49 | ret := &NodeInfo{ 50 | name: node.Name, 51 | node: node, 52 | devs: devMap, 53 | deviceCount: deviceCount, 54 | totalMemory: nodeTotalMemory, 55 | } 56 | 57 | // According to the pods' annotations, construct the node allocation 58 | // state 59 | for _, pod := range pods { 60 | for i, c := range pod.Spec.Containers { 61 | predicateIndexes, err := util.GetPredicateIdxOfContainer(pod, i) 62 | if err != nil { 63 | continue 64 | } 65 | for _, index := range predicateIndexes { 66 | var vcore, vmemory uint 67 | if index >= deviceCount { 68 | klog.Infof("invalid predicateIndex %d larger than device count", index) 69 | continue 70 | } 71 | vcore = util.GetGPUResourceOfContainer(&c, util.VCoreAnnotation) 72 | if vcore < util.HundredCore { 73 | vmemory = util.GetGPUResourceOfContainer(&c, util.VMemoryAnnotation) 74 | } else { 75 | vcore = util.HundredCore 76 | vmemory = deviceTotalMemory 77 | } 78 | err = ret.AddUsedResources(index, vcore, vmemory) 79 | if err != nil { 80 | klog.Infof("failed to update used resource for node %s dev %d due to %v", 81 | node.Name, index, err) 82 | } 83 | } 84 | 85 | } 86 | } 87 | 88 | return ret 89 | } 90 | 91 | // AddUsedResources records the used GPU core and memory 92 | func (n *NodeInfo) AddUsedResources(devID int, vcore uint, vmemory uint) error { 93 | err := n.devs[devID].AddUsedResources(vcore, vmemory) 94 | if err != nil { 95 | klog.Infof("failed to update used resource for node %s dev %d due to %v", n.name, devID, err) 96 | return err 97 | } 98 | n.usedCore += vcore 99 | n.usedMemory += vmemory 100 | return nil 101 | } 102 | 103 | // GetDeviceCount returns the number of GPU devices 104 | func (n *NodeInfo) GetDeviceCount() int { 105 | return n.deviceCount 106 | } 107 | 108 | // GetDeviceMap returns each GPU device information structure 109 | func (n *NodeInfo) GetDeviceMap() map[int]*DeviceInfo { 110 | return n.devs 111 | } 112 | 113 | // GetNode returns the original node structure of kubernetes 114 | func (n *NodeInfo) GetNode() *v1.Node { 115 | return n.node 116 | } 117 | 118 | // GetName returns node name 119 | func (n *NodeInfo) GetName() string { 120 | return n.name 121 | } 122 | 123 | // GetAvailableCore returns the remaining cores of this node 124 | func (n *NodeInfo) GetAvailableCore() int { 125 | return n.deviceCount*util.HundredCore - int(n.usedCore) 126 | } 127 | 128 | // GetAvailableMemory returns the remaining memory of this node 129 | func (n *NodeInfo) GetAvailableMemory() int { 130 | return int(n.totalMemory - n.usedMemory) 131 | } 132 | 133 | type nodeInfoPriority struct { 134 | data []*NodeInfo 135 | less []LessFunc 136 | } 137 | 138 | func NodeInfoSort(less ...LessFunc) *nodeInfoPriority { 139 | return &nodeInfoPriority{ 140 | less: less, 141 | } 142 | } 143 | 144 | func (nip *nodeInfoPriority) Sort(data []*NodeInfo) { 145 | nip.data = data 146 | sort.Sort(nip) 147 | } 148 | 149 | func (nip *nodeInfoPriority) Len() int { 150 | return len(nip.data) 151 | } 152 | 153 | func (nip *nodeInfoPriority) Swap(i, j int) { 154 | nip.data[i], nip.data[j] = nip.data[j], nip.data[i] 155 | } 156 | 157 | func (nip *nodeInfoPriority) Less(i, j int) bool { 158 | var k int 159 | 160 | for k = 0; k < len(nip.less)-1; k++ { 161 | less := nip.less[k] 162 | switch { 163 | case less(nip.data[i], nip.data[j]): 164 | return true 165 | case less(nip.data[j], nip.data[i]): 166 | return false 167 | } 168 | } 169 | 170 | return nip.less[k](nip.data[i], nip.data[j]) 171 | } 172 | -------------------------------------------------------------------------------- /pkg/predicate/gpu_predicate_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package predicate 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strconv" 23 | "testing" 24 | "time" 25 | 26 | "tkestack.io/gpu-admission/pkg/util" 27 | 28 | corev1 "k8s.io/api/core/v1" 29 | "k8s.io/apimachinery/pkg/api/resource" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | k8stypes "k8s.io/apimachinery/pkg/types" 32 | "k8s.io/client-go/kubernetes/fake" 33 | ) 34 | 35 | type podRawInfo struct { 36 | Name string 37 | UID string 38 | Containers []containerRawInfo 39 | } 40 | 41 | type containerRawInfo struct { 42 | Name string 43 | Cores int 44 | Memory int 45 | } 46 | 47 | const ( 48 | deviceCount = 2 49 | totalMemory = 8 50 | namespace = "test-ns" 51 | ) 52 | 53 | func TestDeviceFilter(t *testing.T) { 54 | k8sClient := fake.NewSimpleClientset() 55 | gpuFilter, err := NewGPUFilter(k8sClient) 56 | if err != nil { 57 | t.Fatalf("failed to create new gpuFilter due to %v", err) 58 | } 59 | 60 | nodeList := []corev1.Node{} 61 | for i := 0; i < 3; i++ { 62 | n := corev1.Node{ 63 | ObjectMeta: metav1.ObjectMeta{ 64 | Name: "testnode" + strconv.Itoa(i), 65 | }, 66 | Status: corev1.NodeStatus{ 67 | Capacity: corev1.ResourceList{ 68 | util.VCoreAnnotation: resource.MustParse(fmt.Sprintf("%d", deviceCount*util.HundredCore)), 69 | util.VMemoryAnnotation: resource.MustParse(fmt.Sprintf("%d", totalMemory)), 70 | }, 71 | }, 72 | } 73 | nodeList = append(nodeList, n) 74 | } 75 | testCases := []podRawInfo{ 76 | { 77 | Name: "pod-0", 78 | UID: "uid-0", 79 | Containers: []containerRawInfo{ 80 | { 81 | Name: "container-0", 82 | Cores: 10, 83 | Memory: 1, 84 | }, 85 | { 86 | Name: "container-1", 87 | Cores: 10, 88 | Memory: 1, 89 | }, 90 | }, 91 | }, 92 | { 93 | Name: "pod-1", 94 | UID: "uid-1", 95 | Containers: []containerRawInfo{ 96 | { 97 | Name: "container-0", 98 | Cores: 100, 99 | Memory: 3, 100 | }, 101 | { 102 | Name: "container-1", 103 | Cores: 80, 104 | Memory: 3, 105 | }, 106 | }, 107 | }, 108 | { 109 | Name: "pod-2", 110 | UID: "uid-2", 111 | Containers: []containerRawInfo{ 112 | { 113 | Name: "container-0", 114 | Cores: 200, 115 | Memory: 10, 116 | }, 117 | { 118 | Name: "container-without-gpu", 119 | }, 120 | }, 121 | }, 122 | { 123 | Name: "pod-3", 124 | UID: "uid-3", 125 | Containers: []containerRawInfo{ 126 | { 127 | Name: "container-1", 128 | Cores: 10, 129 | Memory: 2, 130 | }, 131 | { 132 | Name: "container-without-gpu", 133 | }, 134 | }, 135 | }, 136 | } 137 | 138 | testResults := []struct { 139 | nodeName string 140 | }{ 141 | { 142 | nodeName: "testnode0", 143 | }, 144 | { 145 | nodeName: "testnode1", 146 | }, 147 | { 148 | nodeName: "testnode2", 149 | }, 150 | { 151 | nodeName: "testnode0", 152 | }, 153 | } 154 | 155 | for i, cs := range testCases { 156 | containers := []corev1.Container{} 157 | for _, c := range cs.Containers { 158 | container := corev1.Container{ 159 | Name: c.Name, 160 | Resources: corev1.ResourceRequirements{ 161 | Limits: corev1.ResourceList{ 162 | util.VCoreAnnotation: resource.MustParse(fmt.Sprintf("%d", c.Cores)), 163 | util.VMemoryAnnotation: resource.MustParse(fmt.Sprintf("%d", c.Memory)), 164 | }, 165 | }, 166 | } 167 | containers = append(containers, container) 168 | } 169 | pod := &corev1.Pod{ 170 | ObjectMeta: metav1.ObjectMeta{ 171 | Name: cs.Name, 172 | UID: k8stypes.UID(cs.UID), 173 | Annotations: make(map[string]string), 174 | }, 175 | Spec: corev1.PodSpec{ 176 | Containers: containers, 177 | }, 178 | Status: corev1.PodStatus{ 179 | Phase: corev1.PodPending, 180 | }, 181 | } 182 | pod, _ = k8sClient.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{}) 183 | 184 | // wait for podLister to sync 185 | time.Sleep(time.Second * 2) 186 | 187 | nodes, failedNodes, err := gpuFilter.deviceFilter(pod, nodeList) 188 | if err != nil { 189 | t.Fatalf("deviceFilter return err: %v", err) 190 | } 191 | if len(nodes) != 1 { 192 | t.Fatalf("deviceFilter should return exact one node: %v, failedNodes: %v", nodes, failedNodes) 193 | } 194 | 195 | if nodes[0].Name != testResults[i].nodeName { 196 | t.Fatalf("choose the wrong node: %s, expect: %s", nodes[0].Name, testResults[i].nodeName) 197 | } 198 | // wait for podLister to sync 199 | time.Sleep(time.Second * 2) 200 | 201 | // get the latest pod and bind it to the node 202 | pod, _ = k8sClient.CoreV1().Pods(namespace).Get(context.Background(), pod.Name, metav1.GetOptions{}) 203 | pod.Spec.NodeName = nodes[0].Name 204 | pod.Status.Phase = corev1.PodRunning 205 | pod, _ = k8sClient.CoreV1().Pods("test-ns").Update(context.Background(), pod, metav1.UpdateOptions{}) 206 | } 207 | 208 | } 209 | -------------------------------------------------------------------------------- /hack/lib/version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | readonly GO_PACKAGE=tkestack.io/gaiastack/gpu-admission 4 | 5 | # ----------------------------------------------------------------------------- 6 | # Version management helpers. These functions help to set, save and load the 7 | # following variables: 8 | # 9 | # GIT_COMMIT - The git commit id corresponding to this 10 | # source code. 11 | # GIT_TREE_STATE - "clean" indicates no changes since the git commit id 12 | # "dirty" indicates source code changes after the git commit id 13 | # "archive" indicates the tree was produced by 'git archive' 14 | # GIT_VERSION - "vX.Y" used to indicate the last release version. 15 | # GIT_MAJOR - The major part of the version 16 | # GIT_MINOR - The minor component of the version 17 | 18 | # Grovels through git to set a set of env variables. 19 | # 20 | # If GIT_VERSION_FILE, this function will load from that file instead of 21 | # querying git. 22 | api::version::get_version_vars() { 23 | if [[ -n ${GIT_VERSION_FILE-} ]] && [[ -f ${GIT_VERSION_FILE-} ]]; then 24 | api::version::load_version_vars "${GIT_VERSION_FILE}" 25 | return 26 | fi 27 | 28 | # If the gpu-admission source was exported through git archive, then 29 | # we likely don't have a git tree, but these magic values may be filled in. 30 | if [[ '$Format:%%$' == "%" ]]; then 31 | GIT_COMMIT='$Format:%H$' 32 | GIT_TREE_STATE="archive" 33 | # When a 'git archive' is exported, the '$Format:%D$' below will look 34 | # something like 'HEAD -> release-1.8, tag: v1.8.3' where then 'tag: ' 35 | # can be extracted from it. 36 | if [[ '$Format:%D$' =~ tag:\ (v[^ ]+) ]]; then 37 | GIT_VERSION="${BASH_REMATCH[1]}" 38 | fi 39 | fi 40 | 41 | local git=(git --work-tree "${ROOT}") 42 | 43 | if [[ -n ${GIT_COMMIT-} ]] || GIT_COMMIT=$("${git[@]}" rev-parse "HEAD^{commit}" 2>/dev/null); then 44 | if [[ -z ${GIT_TREE_STATE-} ]]; then 45 | # Check if the tree is dirty. default to dirty 46 | if git_status=$("${git[@]}" status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then 47 | GIT_TREE_STATE="clean" 48 | else 49 | GIT_TREE_STATE="dirty" 50 | fi 51 | fi 52 | 53 | # Use git branch to find the branch of current commit. 54 | if [[ -z ${GIT_BRANCH-} ]] ; then 55 | GIT_BRANCH=$("${git[@]}" branch | sed -n -e 's/^\* \(.*\)/\1/p' | tr -d "[:blank:]" 2>/dev/null) 56 | fi 57 | 58 | # Use git describe to find the version based on annotated tags. 59 | if [[ -n ${GIT_VERSION-} ]] || GIT_VERSION=$("${git[@]}" describe --tags --abbrev=14 "${GIT_COMMIT}^{commit}" 2>/dev/null); then 60 | # This translates the "git describe" to an actual semver.org 61 | # compatible semantic version that looks something like this: 62 | # v1.1.0-alpha.0.6+84c76d1142ea4d 63 | # 64 | # TODO: We continue calling this "git version" because so many 65 | # downstream consumers are expecting it there. 66 | DASHES_IN_VERSION=$(echo "${GIT_VERSION}" | sed "s/[^-]//g") 67 | if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then 68 | # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) 69 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\+\2/") 70 | elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then 71 | # We have distance to base tag (v1.1.0-1-gCommitHash) 72 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/+\1/") 73 | fi 74 | if [[ "${GIT_TREE_STATE}" == "dirty" ]]; then 75 | # git describe --dirty only considers changes to existing files, but 76 | # that is problematic since new untracked .go files affect the build, 77 | # so use our idea of "dirty" from git status instead. 78 | GIT_VERSION+="-dirty" 79 | fi 80 | 81 | # Try to match the "git describe" output to a regex to try to extract 82 | # the "major" and "minor" versions and whether this is the exact tagged 83 | # version or whether the tree is between two tagged versions. 84 | if [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?$ ]]; then 85 | GIT_MAJOR=${BASH_REMATCH[1]} 86 | GIT_MINOR=${BASH_REMATCH[2]} 87 | if [[ -n "${BASH_REMATCH[4]}" ]]; then 88 | GIT_MINOR+="+" 89 | fi 90 | fi 91 | fi 92 | fi 93 | } 94 | 95 | # Saves the environment flags to $1 96 | api::version::save_version_vars() { 97 | local version_file=${1-} 98 | [[ -n ${version_file} ]] || { 99 | echo "!!! Internal error. No file specified in api::version::save_version_vars" 100 | return 1 101 | } 102 | 103 | cat <"${version_file}" 104 | GIT_BRANCH='${GIT_BRANCH-}' 105 | GIT_COMMIT='${GIT_COMMIT-}' 106 | GIT_TREE_STATE='${GIT_TREE_STATE-}' 107 | GIT_VERSION='${GIT_VERSION-}' 108 | GIT_MAJOR='${GIT_MAJOR-}' 109 | GIT_MINOR='${GIT_MINOR-}' 110 | EOF 111 | } 112 | 113 | # Loads up the version variables from file $1 114 | api::version::load_version_vars() { 115 | local version_file=${1-} 116 | [[ -n ${version_file} ]] || { 117 | echo "!!! Internal error. No file specified in api::version::load_version_vars" 118 | return 1 119 | } 120 | 121 | source "${version_file}" 122 | } 123 | 124 | api::version::ldflag() { 125 | local key=${1} 126 | local val=${2} 127 | 128 | # If you update these, also update the list pkg/version/def.bzl. 129 | echo "-X ${GO_PACKAGE}/pkg/version.${key}=${val}" 130 | } 131 | 132 | # Prints the value that needs to be passed to the -ldflags parameter of go build 133 | # in order to set the Kubernetes based on the git tree status. 134 | # IMPORTANT: if you update any of these, also update the lists in 135 | # pkg/version/def.bzl and hack/print-workspace-status.sh. 136 | api::version::ldflags() { 137 | api::version::get_version_vars 138 | 139 | local buildDate= 140 | [[ -z ${SOURCE_DATE_EPOCH-} ]] || buildDate="--date=@${SOURCE_DATE_EPOCH}" 141 | local -a ldflags=($(api::version::ldflag "buildDate" "$(date ${buildDate} -u +'%Y-%m-%dT%H:%M:%SZ')")) 142 | if [[ -n ${GIT_BRANCH-} ]]; then 143 | ldflags+=($(api::version::ldflag "gitBranch" "${GIT_BRANCH}")) 144 | fi 145 | 146 | if [[ -n ${GIT_COMMIT-} ]]; then 147 | ldflags+=($(api::version::ldflag "gitCommit" "${GIT_COMMIT}")) 148 | ldflags+=($(api::version::ldflag "gitTreeState" "${GIT_TREE_STATE}")) 149 | fi 150 | 151 | if [[ -n ${GIT_VERSION-} ]]; then 152 | ldflags+=($(api::version::ldflag "gitVersion" "${GIT_VERSION}")) 153 | fi 154 | 155 | if [[ -n ${GIT_MAJOR-} && -n ${GIT_MINOR-} ]]; then 156 | ldflags+=( 157 | $(api::version::ldflag "gitMajor" "${GIT_MAJOR}") 158 | $(api::version::ldflag "gitMinor" "${GIT_MINOR}") 159 | ) 160 | fi 161 | 162 | # The -ldflags parameter takes a single string, so join the output. 163 | echo "${ldflags[*]-}" 164 | } 165 | -------------------------------------------------------------------------------- /pkg/predicate/gpu_predicate.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack available. 3 | * 4 | * Copyright (C) 2012-2019 Tencent. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | package predicate 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "strings" 24 | "time" 25 | 26 | corev1 "k8s.io/api/core/v1" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/labels" 29 | k8stypes "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/apimachinery/pkg/util/wait" 31 | kubeinformers "k8s.io/client-go/informers" 32 | "k8s.io/client-go/kubernetes" 33 | listerv1 "k8s.io/client-go/listers/core/v1" 34 | "k8s.io/klog" 35 | extenderv1 "k8s.io/kube-scheduler/extender/v1" 36 | 37 | "tkestack.io/gpu-admission/pkg/algorithm" 38 | "tkestack.io/gpu-admission/pkg/device" 39 | "tkestack.io/gpu-admission/pkg/util" 40 | ) 41 | 42 | type GPUFilter struct { 43 | kubeClient kubernetes.Interface 44 | nodeLister listerv1.NodeLister 45 | podLister listerv1.PodLister 46 | } 47 | 48 | const ( 49 | NAME = "GPUPredicate" 50 | PodPhaseField = "status.phase" 51 | waitTimeout = 10 * time.Second 52 | ) 53 | 54 | func NewGPUFilter(client kubernetes.Interface) (*GPUFilter, error) { 55 | nodeInformerFactory := kubeinformers.NewSharedInformerFactory(client, time.Second*30) 56 | 57 | podListOptions := func(options *metav1.ListOptions) { 58 | options.FieldSelector = fmt.Sprintf("%s!=%s", PodPhaseField, corev1.PodSucceeded) 59 | } 60 | podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(client, 61 | time.Second*30, kubeinformers.WithNamespace(metav1.NamespaceAll), 62 | kubeinformers.WithTweakListOptions(podListOptions)) 63 | 64 | gpuFilter := &GPUFilter{ 65 | kubeClient: client, 66 | nodeLister: nodeInformerFactory.Core().V1().Nodes().Lister(), 67 | podLister: podInformerFactory.Core().V1().Pods().Lister(), 68 | } 69 | 70 | go nodeInformerFactory.Start(nil) 71 | go podInformerFactory.Start(nil) 72 | 73 | return gpuFilter, nil 74 | } 75 | 76 | func (gpuFilter *GPUFilter) Name() string { 77 | return NAME 78 | } 79 | 80 | type filterFunc func(*corev1.Pod, []corev1.Node) ([]corev1.Node, extenderv1.FailedNodesMap, 81 | error) 82 | 83 | func (gpuFilter *GPUFilter) Filter( 84 | args extenderv1.ExtenderArgs, 85 | ) *extenderv1.ExtenderFilterResult { 86 | if !util.IsGPURequiredPod(args.Pod) { 87 | return &extenderv1.ExtenderFilterResult{ 88 | Nodes: args.Nodes, 89 | FailedNodes: nil, 90 | Error: "", 91 | } 92 | } 93 | 94 | filters := []filterFunc{ 95 | gpuFilter.deviceFilter, 96 | } 97 | filteredNodes := args.Nodes.Items 98 | failedNodesMap := make(extenderv1.FailedNodesMap) 99 | for _, filter := range filters { 100 | passedNodes, failedNodes, err := filter(args.Pod, filteredNodes) 101 | if err != nil { 102 | return &extenderv1.ExtenderFilterResult{ 103 | Error: err.Error(), 104 | } 105 | } 106 | filteredNodes = passedNodes 107 | for name, reason := range failedNodes { 108 | failedNodesMap[name] = reason 109 | } 110 | } 111 | 112 | return &extenderv1.ExtenderFilterResult{ 113 | Nodes: &corev1.NodeList{ 114 | Items: filteredNodes, 115 | }, 116 | FailedNodes: failedNodesMap, 117 | Error: "", 118 | } 119 | } 120 | 121 | //deviceFilter will choose one and only one node fullfil the request, 122 | //so it should always be the last filter of gpuFilter 123 | func (gpuFilter *GPUFilter) deviceFilter( 124 | pod *corev1.Pod, nodes []corev1.Node) ([]corev1.Node, extenderv1.FailedNodesMap, error) { 125 | // #lizard forgives 126 | var ( 127 | filteredNodes = make([]corev1.Node, 0) 128 | failedNodesMap = make(extenderv1.FailedNodesMap) 129 | nodeInfoList []*device.NodeInfo 130 | success bool 131 | sorter = device.NodeInfoSort( 132 | device.ByAllocatableCores, 133 | device.ByAllocatableMemory, 134 | device.ByID) 135 | ) 136 | for k := range pod.Annotations { 137 | if strings.Contains(k, util.GPUAssigned) || 138 | strings.Contains(k, util.PredicateTimeAnnotation) || 139 | strings.Contains(k, util.PredicateGPUIndexPrefix) { 140 | return filteredNodes, failedNodesMap, fmt.Errorf("pod %s had been predicated!", pod.Name) 141 | } 142 | } 143 | 144 | for i := range nodes { 145 | node := &nodes[i] 146 | if !util.IsGPUEnabledNode(node) { 147 | failedNodesMap[node.Name] = "no GPU device" 148 | continue 149 | } 150 | pods, err := gpuFilter.ListPodsOnNode(node) 151 | if err != nil { 152 | failedNodesMap[node.Name] = "failed to get pods on node" 153 | continue 154 | } 155 | nodeInfo := device.NewNodeInfo(node, pods) 156 | nodeInfoList = append(nodeInfoList, nodeInfo) 157 | } 158 | sorter.Sort(nodeInfoList) 159 | 160 | for _, nodeInfo := range nodeInfoList { 161 | node := nodeInfo.GetNode() 162 | if success { 163 | failedNodesMap[node.Name] = fmt.Sprintf( 164 | "pod %s has already been matched to another node", pod.UID) 165 | continue 166 | } 167 | 168 | alloc := algorithm.NewAllocator(nodeInfo) 169 | newPod, err := alloc.Allocate(pod) 170 | if err != nil { 171 | failedNodesMap[node.Name] = fmt.Sprintf( 172 | "pod %s does not match with this node", pod.UID) 173 | continue 174 | } else { 175 | annotationMap := make(map[string]string) 176 | for k, v := range newPod.Annotations { 177 | if strings.Contains(k, util.GPUAssigned) || 178 | strings.Contains(k, util.PredicateTimeAnnotation) || 179 | strings.Contains(k, util.PredicateGPUIndexPrefix) || 180 | strings.Contains(k, util.PredicateNode) { 181 | annotationMap[k] = v 182 | } 183 | } 184 | err := gpuFilter.patchPodWithAnnotations(newPod, annotationMap) 185 | if err != nil { 186 | failedNodesMap[node.Name] = "update pod annotation failed" 187 | continue 188 | } 189 | filteredNodes = append(filteredNodes, *node) 190 | success = true 191 | } 192 | } 193 | 194 | return filteredNodes, failedNodesMap, nil 195 | } 196 | 197 | func (gpuFilter *GPUFilter) ListPodsOnNode(node *corev1.Node) ([]*corev1.Pod, error) { 198 | // #lizard forgives 199 | pods, err := gpuFilter.podLister.Pods(corev1.NamespaceAll).List(labels.Everything()) 200 | if err != nil { 201 | return nil, err 202 | } 203 | 204 | var ret []*corev1.Pod 205 | for _, pod := range pods { 206 | klog.V(9).Infof("List pod %s", pod.Name) 207 | var predicateNode string 208 | if pod.Spec.NodeName == "" && pod.Annotations != nil { 209 | if v, ok := pod.Annotations[util.PredicateNode]; ok { 210 | predicateNode = v 211 | } 212 | } 213 | if (pod.Spec.NodeName == node.Name || predicateNode == node.Name) && 214 | pod.Status.Phase != corev1.PodSucceeded && 215 | pod.Status.Phase != corev1.PodFailed { 216 | ret = append(ret, pod) 217 | klog.V(9).Infof("get pod %s on node %s", pod.UID, node.Name) 218 | } 219 | } 220 | return ret, nil 221 | } 222 | 223 | func (gpuFilter *GPUFilter) patchPodWithAnnotations( 224 | pod *corev1.Pod, annotationMap map[string]string) error { 225 | // update annotations by patching to the pod 226 | type patchMetadata struct { 227 | Annotations map[string]string `json:"annotations"` 228 | } 229 | type patchPod struct { 230 | Metadata patchMetadata `json:"metadata"` 231 | } 232 | payload := patchPod{ 233 | Metadata: patchMetadata{ 234 | Annotations: annotationMap, 235 | }, 236 | } 237 | 238 | payloadBytes, _ := json.Marshal(payload) 239 | err := wait.PollImmediate(time.Second, waitTimeout, func() (bool, error) { 240 | _, err := gpuFilter.kubeClient.CoreV1().Pods(pod.Namespace). 241 | Patch(context.Background(), pod.Name, k8stypes.StrategicMergePatchType, payloadBytes, metav1.PatchOptions{}) 242 | if err == nil { 243 | return true, nil 244 | } 245 | if util.ShouldRetry(err) { 246 | return false, nil 247 | } 248 | 249 | return false, err 250 | }) 251 | if err != nil { 252 | msg := fmt.Sprintf("failed to add annotation %v to pod %s due to %s", 253 | annotationMap, pod.UID, err.Error()) 254 | klog.Infof(msg) 255 | return fmt.Errorf(msg) 256 | } 257 | return nil 258 | } 259 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM= 2 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 4 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 5 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 6 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 7 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 8 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 9 | cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= 10 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 11 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 12 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 13 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 14 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 15 | github.com/Azure/azure-sdk-for-go v21.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= 16 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= 17 | github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= 18 | github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= 19 | github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= 20 | github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= 21 | github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= 22 | github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= 23 | github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= 24 | github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 25 | github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 26 | github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= 27 | github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= 28 | github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= 29 | github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 30 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 31 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 32 | github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20181220005116-f8e995905100/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= 33 | github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= 34 | github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= 35 | github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= 36 | github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= 37 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 38 | github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 39 | github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 40 | github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 41 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 42 | github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg= 43 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 44 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 45 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 46 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 47 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 48 | github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 49 | github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM= 50 | github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 51 | github.com/bazelbuild/bazel-gazelle v0.0.0-20181012220611-c728ce9f663e/go.mod h1:uHBSeeATKpVazAACZBDPL/Nk/UhQDDsJWDlqYJo8/Us= 52 | github.com/bazelbuild/buildtools v0.0.0-20180226164855-80c7f0d45d7e/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= 53 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 54 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 55 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 56 | github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= 57 | github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= 58 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 59 | github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= 60 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 61 | github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= 62 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 63 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 64 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 65 | github.com/client9/misspell v0.0.0-20170928000206-9ce5d979ffda/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 66 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 67 | github.com/cloudflare/cfssl v0.0.0-20180726162950-56268a613adf/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= 68 | github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= 69 | github.com/codedellemc/goscaleio v0.0.0-20170830184815-20e2ce2cf885/go.mod h1:JIHmDHNZO4tmA3y3RHp6+Gap6kFsNf55W9Pn/3YS9IY= 70 | github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= 71 | github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= 72 | github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= 73 | github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= 74 | github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= 75 | github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= 76 | github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 77 | github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 78 | github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= 79 | github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 80 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 81 | github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 82 | github.com/coreos/rkt v1.30.0/go.mod h1:O634mlH6U7qk87poQifK6M2rsFNt+FyUTWNMnP1hF1U= 83 | github.com/cpuguy83/go-md2man v1.0.4/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= 84 | github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= 85 | github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= 86 | github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= 87 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 88 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 89 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 90 | github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= 91 | github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 92 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 93 | github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= 94 | github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 95 | github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 96 | github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 97 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 98 | github.com/docker/libnetwork v0.0.0-20180830151422-a9cd636e3789/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= 99 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= 100 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= 101 | github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 102 | github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 103 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 104 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 105 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 106 | github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= 107 | github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac= 108 | github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 109 | github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= 110 | github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 111 | github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= 112 | github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= 113 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 114 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 115 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 116 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 117 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 118 | github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 119 | github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 120 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 121 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 122 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 123 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 124 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 125 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 126 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 127 | github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= 128 | github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= 129 | github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= 130 | github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 131 | github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 132 | github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 133 | github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 134 | github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= 135 | github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 136 | github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 137 | github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= 138 | github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 139 | github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 140 | github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 141 | github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 142 | github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= 143 | github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q= 144 | github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= 145 | github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 146 | github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 147 | github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 148 | github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= 149 | github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 150 | github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 151 | github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= 152 | github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= 153 | github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= 154 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 155 | github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= 156 | github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 157 | github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= 158 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 159 | github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= 160 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 161 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 162 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 163 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= 164 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 165 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 166 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= 167 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 168 | github.com/golang/mock v0.0.0-20160127222235-bd3c8e81be01/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 169 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 170 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 171 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 172 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 173 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 174 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 175 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 176 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 177 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 178 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 179 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 180 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 181 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 182 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 183 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 184 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 185 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 186 | github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= 187 | github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= 188 | github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= 189 | github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 190 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 191 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 192 | github.com/google/cadvisor v0.33.2-0.20190411163913-9db8c7dee20a/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48= 193 | github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= 194 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 195 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 196 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 197 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 198 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 199 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 200 | github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 201 | github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= 202 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 203 | github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= 204 | github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 205 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 206 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 207 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 208 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 209 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 210 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 211 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 212 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 213 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 214 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 215 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= 216 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 217 | github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= 218 | github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 219 | github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= 220 | github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= 221 | github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= 222 | github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= 223 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 224 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 225 | github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 226 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 227 | github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 228 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 229 | github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 230 | github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 231 | github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= 232 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 233 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 234 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 235 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 236 | github.com/hashicorp/hcl v0.0.0-20160711231752-d8c773c4cba1/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= 237 | github.com/heketi/heketi v0.0.0-20181109135656-558b29266ce0/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= 238 | github.com/heketi/rest v0.0.0-20180404230133-aa6a65207413/go.mod h1:BeS3M108VzVlmAue3lv2WcGuPAX94/KN63MUURzbYSI= 239 | github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= 240 | github.com/heketi/utils v0.0.0-20170317161834-435bc5bdfa64/go.mod h1:RYlF4ghFZPPmk2TC5REt5OFwvfb6lzxFWrTWB+qs28s= 241 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 242 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 243 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 244 | github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= 245 | github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 246 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 247 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 248 | github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 249 | github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 250 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 251 | github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= 252 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 253 | github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= 254 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 255 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 256 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 257 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 258 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 259 | github.com/jteeuwen/go-bindata v0.0.0-20151023091102-a0ff2567cfb7/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs= 260 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 261 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 262 | github.com/julienschmidt/httprouter v1.3.1-0.20191005171706-08a3b3d20bbe h1:cyWTfIC9qUdYm5OEEhOHKkXf5mWvEgooCa6lsMozVKk= 263 | github.com/julienschmidt/httprouter v1.3.1-0.20191005171706-08a3b3d20bbe/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 264 | github.com/kardianos/osext v0.0.0-20150410034420-8fef92e41e22/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= 265 | github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= 266 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 267 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 268 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 269 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 270 | github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169/go.mod h1:glhvuHOU9Hy7/8PwwdtnarXqLagOX0b/TbZx2zLMqEg= 271 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 272 | github.com/kr/pretty v0.0.0-20140812000539-f31442d60e51/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA= 273 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 274 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 275 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 276 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 277 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 278 | github.com/kr/text v0.0.0-20130911015532-6807e777504f/go.mod h1:sjUstKUATFIcff4qlB53Kml0wQPtJVc/3fWrmuUmcfA= 279 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 280 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 281 | github.com/kubernetes/kubernetes/staging/src/k8s.io/api v0.0.0-20190816231410-2d3c76f9091b h1:3vuFbOklcXS2ZE0HmTHCWJ7iGaQ45P5XayawL30R9qQ= 282 | github.com/kubernetes/kubernetes/staging/src/k8s.io/api v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:rcBmQEBoKrTUCORrHN/yvdmJPQsGpCEL61sZkMpMX/8= 283 | github.com/kubernetes/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:F6Fl77o501YUXNsJfBI+WAoC0ZcVGbw3FWQYig2Eplw= 284 | github.com/kubernetes/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190816231410-2d3c76f9091b h1:aZcokGBj4Pi6cSljbstuiZoko14xjqa+or2HUqKrtMo= 285 | github.com/kubernetes/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:EM3ldevXE4F3MtIQ4hiuD/tCPbERnIGmMbh/rRhJTWU= 286 | github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190816231410-2d3c76f9091b h1:qGnMUPQtVHWUhRCqwiBgvC39nMQSmvgr7ziWlcaCw7M= 287 | github.com/kubernetes/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:OZJzR5V6BBBN4xOUBmqtPhb4RuJez+IWLMODK/NIlJQ= 288 | github.com/kubernetes/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:dKWyWMnUIiQfD5yrNxeI07y0i6s19N9qnMId+knbLPI= 289 | github.com/kubernetes/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190816231410-2d3c76f9091b h1:GxEP/vVCcMU/30ejrtbyvVzYsp0cGnyNTMaW+nEHaNE= 290 | github.com/kubernetes/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:NDJvuHvuvLV6miyzCs9HtkqBxy1Z+6UhEPYnUx38+6E= 291 | github.com/kubernetes/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:861E8pSdrE1y4su5sU2ybvnPMpymHFWZtnFl75mWktE= 292 | github.com/kubernetes/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:r5Ddw/Lh5GppfYcOPMtWU06QKtXaHj6iPHSZ3RZeJGU= 293 | github.com/kubernetes/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:4Gc8gg/oUtfQgnOvrhYAu1AEwEpSzP8er8bKHbjVJBo= 294 | github.com/kubernetes/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190816231410-2d3c76f9091b h1:2btyNDkZ7SXGCH+TdbqZJekySU+uvZRS0u0ZY90uuYg= 295 | github.com/kubernetes/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:NJRBXyb9zH0JrIobSBvZBoqUyxFXxcm0bN7Qr6MN12k= 296 | github.com/kubernetes/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:NXoFYaThKj0dmWgmQCs5fyce69w7dSPgJSjopV7HbwM= 297 | github.com/kubernetes/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:xwK4w9WjvsWqe6BcCPgj/LRCLt0DyP2z0jLzkskrqk4= 298 | github.com/kubernetes/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:B0gdwJ6Lc3iB67M8vFyHLt9FMFIFmSuFqmrqlVzYjSM= 299 | github.com/kubernetes/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:ZiwCid/JscSDAF4cNOV+Cq+8t61kYWZoTsjhH8OaguY= 300 | github.com/kubernetes/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:zFgVsUCkS999et21KQUK67gYZkbJbdRY0YJIw6TZyMg= 301 | github.com/kubernetes/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:Q70yepfdw1i1HQkSCMIL1gzcS4keWGjjDPkFIWEGX64= 302 | github.com/kubernetes/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:PtmuzszR3KI3+Ao8TV/Vm2c28uiLf/RT2P8PQR3Ddfg= 303 | github.com/kubernetes/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:nypsLFh31wJ+39/Hc52DBhS3277H3ztzYah1d4J0kaw= 304 | github.com/kubernetes/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:ZXuwfKuK6ZMbreRcmr4gRIa5dfuA+IG5oB+HhpB1Hjo= 305 | github.com/kubernetes/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20190816231410-2d3c76f9091b/go.mod h1:AV4PilnJ+BCoSk8Y3pNti4CEBwVDOyTf1yFmyxj69ng= 306 | github.com/libopenstorage/openstorage v0.0.0-20170906232338-093a0c388875/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= 307 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= 308 | github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= 309 | github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= 310 | github.com/magiconair/properties v0.0.0-20160816085511-61b492c03cf4/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 311 | github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 312 | github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 313 | github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= 314 | github.com/mattn/go-shellwords v0.0.0-20180605041737-f8471b0a71de/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= 315 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 316 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 317 | github.com/mesos/mesos-go v0.0.9/go.mod h1:kPYCMQ9gsOXVAle1OsoY4I1+9kPu8GHkf88aV59fDr4= 318 | github.com/mholt/caddy v0.0.0-20180213163048-2de495001514/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY= 319 | github.com/miekg/dns v0.0.0-20160614162101-5d001d020961/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 320 | github.com/mindprince/gonvml v0.0.0-20171110221305-fee913ce8fb2/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= 321 | github.com/mistifyio/go-zfs v0.0.0-20151009155749-1b4ae6fb4e77/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= 322 | github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= 323 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 324 | github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= 325 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 326 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 327 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 328 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 329 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 330 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 331 | github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= 332 | github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 333 | github.com/mrunalp/fileutils v0.0.0-20160930181131-4ee1cc9a8058/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= 334 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 335 | github.com/mvdan/xurls v0.0.0-20160110113200-1b768d7c393a/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= 336 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 337 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 338 | github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= 339 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 340 | github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= 341 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 342 | github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= 343 | github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 344 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 345 | github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3 h1:EooPXg51Tn+xmWPXJUGCnJhJSpeuMlBmfJVcqIRmmv8= 346 | github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 347 | github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= 348 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 349 | github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 350 | github.com/opencontainers/image-spec v0.0.0-20170604055404-372ad780f634/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 351 | github.com/opencontainers/runc v0.0.0-20181113202123-f000fe11ece1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= 352 | github.com/opencontainers/runtime-spec v1.0.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= 353 | github.com/opencontainers/selinux v0.0.0-20170621221121-4a2974bf1ee9/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= 354 | github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 355 | github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 356 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 357 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 358 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 359 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 360 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 361 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 362 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 363 | github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= 364 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 365 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 366 | github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= 367 | github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= 368 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 369 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 370 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 371 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 372 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 373 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 374 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 375 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 376 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 377 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 378 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 379 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 380 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 381 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 382 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 383 | github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= 384 | github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= 385 | github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= 386 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 387 | github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= 388 | github.com/russross/blackfriday v0.0.0-20151117072312-300106c228d5/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 389 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 390 | github.com/seccomp/libseccomp-golang v0.0.0-20150813023252-1b506fc7c24e/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= 391 | github.com/shurcooL/sanitized_anchor_name v0.0.0-20151028001915-10ef21a441db/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 392 | github.com/sigma/go-inotify v0.0.0-20181102212354-c87b6cf5033d/go.mod h1:stlh9OsqBQSdwxTxX73mu41BBtRbIpZLQ7flcAoxAfo= 393 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 394 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 395 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 396 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 397 | github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 398 | github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 399 | github.com/spf13/afero v0.0.0-20160816080757-b28a7effac97/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 400 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 401 | github.com/spf13/cast v0.0.0-20160730092037-e31f36ffc91a/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= 402 | github.com/spf13/cobra v0.0.0-20180319062004-c439c4fa0937/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 403 | github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 404 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 405 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 406 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 407 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 408 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 409 | github.com/spf13/viper v0.0.0-20160820190039-7fb2782df3d8/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= 410 | github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= 411 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 412 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 413 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 414 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 415 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 416 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 417 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 418 | github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= 419 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 420 | github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 421 | github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= 422 | github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= 423 | github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= 424 | github.com/vmware/photon-controller-go-sdk v0.0.0-20170310013346-4a435daef6cc/go.mod h1:e6humHha1ekIwTCm+A5Qed5mG8V4JL+ChHcUOJ+L/8U= 425 | github.com/xanzy/go-cloudstack v0.0.0-20160728180336-1e2cbf647e57/go.mod h1:s3eL3z5pNXF5FVybcT+LIVdId8pYn709yv6v5mrkrQE= 426 | github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 427 | github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= 428 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 429 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 430 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 431 | go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 432 | go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= 433 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 434 | go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 435 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 436 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 437 | go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 438 | go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= 439 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 440 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 441 | golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 442 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 443 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= 444 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 445 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 446 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 447 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 448 | golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 449 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= 450 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 451 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= 452 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 453 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 454 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 455 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 456 | golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 457 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 458 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 459 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 460 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 461 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 462 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 463 | golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 464 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 465 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 466 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 467 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 468 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 469 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 470 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 471 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 472 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 473 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 474 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 475 | golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 476 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 477 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 478 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 479 | golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 480 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 481 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 482 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 483 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 484 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 485 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 486 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 487 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 488 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 489 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 490 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 491 | golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 492 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= 493 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 494 | golang.org/x/net v0.0.0-20191109021931-daa7c04131f5 h1:bHNaocaoJxYBo5cw41UyTMLjYlb8wPY7+WFrnklbHOM= 495 | golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 496 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 497 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 498 | golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= 499 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 500 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 501 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 502 | golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= 503 | golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 504 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 505 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 506 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= 507 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 508 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 509 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 510 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 511 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 512 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 513 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 514 | golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 515 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 516 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 517 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 518 | golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 519 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 520 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 521 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 522 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 523 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 524 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 525 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 526 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 527 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 528 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 529 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 530 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= 531 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 532 | golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= 533 | golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 534 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 535 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 536 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 537 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 538 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 539 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 540 | golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= 541 | golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 542 | golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 543 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 544 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 545 | golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= 546 | golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 547 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 548 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 549 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 550 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 551 | golang.org/x/time v0.0.0-20161028155119-f51c12702a4d h1:TnM+PKb3ylGmZvyPXmo9m/wktg7Jn/a/fNmr33HSj8g= 552 | golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 553 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 554 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= 555 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 556 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= 557 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 558 | golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 559 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 560 | golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 561 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 562 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 563 | golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 564 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 565 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 566 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 567 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 568 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 569 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 570 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 571 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 572 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 573 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 574 | golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 575 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 576 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 577 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 578 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 579 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 580 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 581 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 582 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 583 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 584 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 585 | gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= 586 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 587 | gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= 588 | google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 589 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 590 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 591 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 592 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 593 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 594 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 595 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 596 | google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= 597 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 598 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 599 | google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= 600 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 601 | google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 602 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 603 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 604 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 605 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 606 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 607 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 608 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 609 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 610 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 611 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 612 | google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 613 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 614 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 615 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 616 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 617 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 618 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 619 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 620 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 621 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 622 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 623 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 624 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 625 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 626 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 627 | google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= 628 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 629 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 630 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 631 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 632 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 633 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 634 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 635 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 636 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 637 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 638 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 639 | gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= 640 | gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= 641 | gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 642 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 643 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 644 | gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 645 | gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= 646 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 647 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 648 | gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 649 | gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= 650 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 651 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 652 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 653 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 654 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 655 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 656 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 657 | gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 658 | gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= 659 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 660 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 661 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 662 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 663 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 664 | k8s.io/api v0.18.12 h1:97X6znOXMVgCKivTAgpBXGBGlCe3gbM++yFdldgBCaE= 665 | k8s.io/api v0.18.12/go.mod h1:3sS78jmUoGHwERyMbEhxP6owcQ77UxGo+Yy+dKNWrh0= 666 | k8s.io/api v0.19.4 h1:I+1I4cgJYuCDgiLNjKx7SLmIbwgj9w7N7Zr5vSIdwpo= 667 | k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= 668 | k8s.io/apimachinery v0.18.12 h1:bLFXU4IxOu06F6Z6PV7eqtapXFb1G2q0ni0XBNFtJH8= 669 | k8s.io/apimachinery v0.18.12/go.mod h1:PF5taHbXgTEJLU+xMypMmYTXTWPJ5LaW8bfsisxnEXk= 670 | k8s.io/apimachinery v0.19.4 h1:+ZoddM7nbzrDCp0T3SWnyxqf8cbWPT2fkZImoyvHUG0= 671 | k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= 672 | k8s.io/client-go v0.18.12 h1:MDGRE2tGidz29g45dI4kfelJo+aRmDqWx0Way8mD88A= 673 | k8s.io/client-go v0.18.12/go.mod h1:0aC8XkA09dX/goYqHQJ/kVv0zL1t+weOZt3pmz9LpxA= 674 | k8s.io/client-go v0.19.4 h1:85D3mDNoLF+xqpyE9Dh/OtrJDyJrSRKkHmDXIbEzer8= 675 | k8s.io/client-go v0.19.4/go.mod h1:ZrEy7+wj9PjH5VMBCuu/BDlvtUAku0oVFk4MmnW9mWA= 676 | k8s.io/component-base v0.18.12 h1:1BfsKoZZgSU1GanVsCDgMkCVPnT0B8iaHV1Ui0ror6Y= 677 | k8s.io/component-base v0.18.12/go.mod h1:pRGKXsx2KWfsJqlDi4sbCc1jpaB87rXIIqupjhr5wj0= 678 | k8s.io/component-base v0.19.4 h1:HobPRToQ8KJ9ubRju6PUAk9I5V1GNMJZ4PyWbiWA0uI= 679 | k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70= 680 | k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 681 | k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 682 | k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 683 | k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM= 684 | k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 685 | k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 686 | k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= 687 | k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 688 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 689 | k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 690 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 691 | k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= 692 | k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= 693 | k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= 694 | k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= 695 | k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= 696 | k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= 697 | k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= 698 | k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= 699 | k8s.io/kube-scheduler v0.18.12 h1:hEWpwQ0Krv4ODYhg0cRcqjgiHCci1T9cW8e1k0+9LXs= 700 | k8s.io/kube-scheduler v0.18.12/go.mod h1:usvx8hz5ws8GSWgOPyRUuVzAhRDQmtOo2RDCkyLFSf4= 701 | k8s.io/kube-scheduler v0.19.4 h1:gADciJ7aQsjDV76P6hto32Z/m3UhcGNXh0VwL+1x3O0= 702 | k8s.io/kube-scheduler v0.19.4/go.mod h1:r0SdH7fwIKyA/MURcRS4jNjTjfBEju3oIBll0Dxqjj4= 703 | k8s.io/kubernetes v1.15.5 h1:joGHGmNjm75F8LkJmflKxaYTHWNwodNp+aN/uAgCNp0= 704 | k8s.io/kubernetes v1.15.5/go.mod h1:Kyu5t/D1Dx5GSj78bquQyN20G5xKZ9BYrtFT4fA9kXU= 705 | k8s.io/kubernetes v1.19.4 h1:3cKXaFFb+2yIBM5jILpuyx6m2dW2cgPcjYbNybhpewo= 706 | k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3/go.mod h1:+G1xBfZDfVFsm1Tj/HNCvg4QqWx8rJ2Fxpqr1rqp/gQ= 707 | k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= 708 | k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= 709 | k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= 710 | k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= 711 | k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= 712 | k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 713 | modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= 714 | modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= 715 | modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= 716 | modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= 717 | modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= 718 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 719 | sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= 720 | sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= 721 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= 722 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= 723 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= 724 | sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= 725 | sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= 726 | sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= 727 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 728 | sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= 729 | sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= 730 | vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= 731 | --------------------------------------------------------------------------------