├── .dockerignore ├── go.mod ├── Dockerfile ├── pkg ├── scaler │ ├── jsonpatch.go │ └── scaler.go ├── kubeconfig │ └── kubeconfig.go └── proxy │ ├── proxy_test.go │ └── proxy.go ├── shell.nix ├── README.md ├── main.go ├── LICENSE └── go.sum /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | README.md 3 | shell.nix -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/greenkeytech/zero-pod-autoscaler 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/spf13/pflag v1.0.5 7 | k8s.io/api v0.17.0 8 | k8s.io/apimachinery v0.17.0 9 | k8s.io/client-go v0.17.0 10 | ) 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:experimental 2 | 3 | FROM golang:1.13.5-alpine3.10 AS builder 4 | 5 | RUN apk update && \ 6 | apk add --no-cache \ 7 | git 8 | 9 | WORKDIR /src/zero-pod-autoscaler 10 | COPY go.mod go.sum /src/zero-pod-autoscaler/ 11 | RUN go mod download 12 | 13 | COPY . ./ 14 | RUN --mount=type=cache,target=/tmp/gocache GOCACHE=/tmp/gocache CGO_ENABLED=0 go build . 15 | 16 | FROM scratch 17 | COPY --from=builder /src/zero-pod-autoscaler/zero-pod-autoscaler /bin/zero-pod-autoscaler 18 | ENTRYPOINT [ "/bin/zero-pod-autoscaler" ] 19 | -------------------------------------------------------------------------------- /pkg/scaler/jsonpatch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GreenKey Technologies 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you 4 | // may not use this file except in compliance with the License. You 5 | // may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | package scaler 15 | 16 | import ( 17 | "strings" 18 | ) 19 | 20 | // JsonPatchEscape escapes a string for use as a path component in a 21 | // JSON Patch by replacing "~" with "~0" and "/" with "~1". 22 | // 23 | // http://jsonpatch.com/ 24 | func JsonPatchEscape(s string) string { 25 | return strings.ReplaceAll(strings.ReplaceAll(s, "~", "~0"), "/", "~1") 26 | } 27 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { nixpkgs ? (fetchTarball https://releases.nixos.org/nixpkgs/nixpkgs-20.03pre212770.cc1ae9f21b9/nixexprs.tar.xz) 2 | }: 3 | 4 | with import nixpkgs {}; 5 | 6 | let 7 | 8 | inherit (darwin.apple_sdk.frameworks) CoreFoundation Security; 9 | 10 | in stdenv.mkDerivation { 11 | name = "shell-env"; 12 | 13 | buildInputs = [ 14 | git 15 | 16 | go 17 | golangci-lint 18 | gotools # guru 19 | ] 20 | 21 | # needed for building with cgo (default `go build`, `go run`, etc.) 22 | ++ stdenv.lib.optionals stdenv.isDarwin [ CoreFoundation Security ]; 23 | 24 | shellHook = '' 25 | unset GOPATH 26 | export GO111MODULE=on 27 | 28 | PATH=$HOME/go/bin:$PATH 29 | 30 | '' + (if stdenv.isDarwin then '' 31 | # https://stackoverflow.com/questions/51161225/how-can-i-make-macos-frameworks-available-to-clang-in-a-nix-environment 32 | export CGO_CFLAGS="-iframework ${CoreFoundation}/Library/Frameworks -iframework ${Security}/Library/Frameworks" 33 | export CGO_LDFLAGS="-F ${CoreFoundation}/Library/Frameworks -F ${Security}/Library/Frameworks -framework CoreFoundation -framework Security" 34 | '' else ""); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /pkg/kubeconfig/kubeconfig.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GreenKey Technologies 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you 4 | // may not use this file except in compliance with the License. You 5 | // may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | package kubeconfig 15 | 16 | import ( 17 | // required to work with clientcmd/api 18 | flag "github.com/spf13/pflag" 19 | 20 | "k8s.io/client-go/kubernetes" 21 | "k8s.io/client-go/tools/clientcmd" 22 | ) 23 | 24 | func BindKubeFlags(flags *flag.FlagSet) (*clientcmd.ClientConfigLoadingRules, *clientcmd.ConfigOverrides) { 25 | loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() 26 | flags.StringVar(&loadingRules.ExplicitPath, "kubeconfig", "", "Path to kubeconfig file to use") 27 | 28 | configOverrides := clientcmd.ConfigOverrides{} 29 | clientcmd.BindOverrideFlags(&configOverrides, flags, clientcmd.RecommendedConfigOverrideFlags("")) 30 | 31 | return loadingRules, &configOverrides 32 | } 33 | 34 | // BuildClientset builds a config that should work both in-cluster 35 | // with no args and out-of-cluster with appropriate args and returns a 36 | // clientset. 37 | func BuildClientset(loadingRules *clientcmd.ClientConfigLoadingRules, configOverrides *clientcmd.ConfigOverrides) (*kubernetes.Clientset, error) { 38 | config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides).ClientConfig() 39 | if err != nil { 40 | return nil, err 41 | } 42 | return kubernetes.NewForConfig(config) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/proxy/proxy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GreenKey Technologies 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you 4 | // may not use this file except in compliance with the License. You 5 | // may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | package proxy 15 | 16 | import ( 17 | "fmt" 18 | "io/ioutil" 19 | "net" 20 | "net/http" 21 | "net/http/httptest" 22 | "testing" 23 | ) 24 | 25 | func proxyOneConnection(t *testing.T, target string) string { 26 | addr := "127.0.0.1:" 27 | ln, err := net.Listen("tcp", addr) 28 | if err != nil { 29 | t.Fatalf("failed to listen on %s: %v", addr, err) 30 | } 31 | 32 | go func() { 33 | defer ln.Close() 34 | 35 | conn, err := ln.Accept() 36 | if err != nil { 37 | t.Fatalf("failed to accept connection: %w", err) 38 | } 39 | 40 | defer conn.Close() 41 | 42 | if err := ProxyTo(conn, target); err != nil { 43 | t.Fatalf("failed to proxy: %v", err) 44 | } 45 | }() 46 | 47 | return ln.Addr().String() 48 | } 49 | 50 | type OkHandler struct { 51 | Body string 52 | } 53 | 54 | func (h OkHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { 55 | w.Write([]byte(h.Body)) 56 | } 57 | 58 | func TestProxyTo(t *testing.T) { 59 | handler := OkHandler{"this is the response body"} 60 | server := httptest.NewServer(handler) 61 | proxyAddr := proxyOneConnection(t, server.Listener.Addr().String()) 62 | 63 | resp, err := http.Get(fmt.Sprintf("http://%s/", proxyAddr)) 64 | if err != nil { 65 | t.Fatalf("failed to get response: %v", err) 66 | } 67 | 68 | defer resp.Body.Close() 69 | body, err := ioutil.ReadAll(resp.Body) 70 | if err != nil { 71 | t.Fatalf("failed to read response body: %v", err) 72 | } 73 | 74 | if string(body) != handler.Body { 75 | t.Fatalf("response body did not match: expected: %q got: %q", handler.Body, body) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /pkg/proxy/proxy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GreenKey Technologies 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you 4 | // may not use this file except in compliance with the License. You 5 | // may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | package proxy 15 | 16 | import ( 17 | "errors" 18 | "fmt" 19 | "io" 20 | "log" 21 | "net" 22 | "syscall" 23 | "time" 24 | ) 25 | 26 | // BiDiCopy copies bidirectional streams. 27 | func BiDiCopy(a, b *net.TCPConn) error { 28 | errch1 := make(chan error) 29 | defer close(errch1) 30 | go func() { 31 | _, err := io.Copy(a, b) 32 | a.CloseWrite() 33 | b.CloseRead() 34 | errch1 <- err 35 | }() 36 | 37 | errch2 := make(chan error) 38 | defer close(errch2) 39 | go func() { 40 | _, err := io.Copy(b, a) 41 | b.CloseWrite() 42 | a.CloseRead() 43 | errch2 <- err 44 | }() 45 | 46 | err1, err2 := <-errch1, <-errch2 47 | 48 | // ignore connection resets because they happen all the time with docker client 49 | if errors.Is(err1, syscall.ECONNRESET) { 50 | err1 = nil 51 | } 52 | if errors.Is(err2, syscall.ECONNRESET) { 53 | err2 = nil 54 | } 55 | 56 | if err1 != nil && err2 != nil { 57 | return fmt.Errorf("failed to copy steams: %w; %w", err1, err2) 58 | } 59 | if err1 != nil { 60 | return fmt.Errorf("failed to copy steam: %w", err1) 61 | } 62 | if err2 != nil { 63 | return fmt.Errorf("failed to copy stream: %w", err2) 64 | } 65 | return nil 66 | } 67 | 68 | // ProxyTo dials a connection to remote then (bi-directionally) copies 69 | // everything from src to the new connection. 70 | func ProxyTo(src net.Conn, remote string) error { 71 | dst, err := net.DialTimeout("tcp", remote, 5*time.Second) 72 | if err != nil { 73 | return err 74 | } 75 | defer dst.Close() 76 | 77 | log.Printf("%s->%s: connected to upstream %s", 78 | src.RemoteAddr(), src.LocalAddr(), remote) 79 | 80 | _dst, ok := dst.(*net.TCPConn) 81 | if !ok { 82 | return fmt.Errorf("dst impossibly not a tcp connection: %T", dst) 83 | } 84 | 85 | _src, ok := src.(*net.TCPConn) 86 | if !ok { 87 | return fmt.Errorf("src not a tcp connection: %T", src) 88 | } 89 | 90 | return BiDiCopy(_dst, _src) 91 | } 92 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Zero Pod Autoscaler 2 | 3 | Zero Pod Autoscaler (ZPA) manages a Deployment resource. It can scale 4 | the Deployment all the way down to zero replicas when it is not in 5 | use. It can work alongside an HPA: when scaled to zero, the HPA 6 | ignores the Deployment; once scaled back to one, the HPA may scale up 7 | further. 8 | 9 | To do this, ZPA is a **TCP proxy** in front of the Service load 10 | balancing a Deployment. This allows the ZPA to track the number of 11 | connections and scale the Deployment to zero when the connection count 12 | has been zero for some time period. 13 | 14 | If scaled to zero, an incoming connection triggers a scale-to-one 15 | action. Once Service's Endpoints include a "ready" address, the 16 | connection can complete, hopefully before the client times out. 17 | 18 | ## Status 19 | 20 | Alpha-quality. Currently deployed successfully for some internal 21 | use-cases. Currently recommended for use only if willing to read 22 | and/or customize the code. 23 | 24 | ## Usage 25 | 26 | Usage is awkward, certainly more difficult than using an HPA. 27 | 28 | The ZPA points to an existing Deployment resource with a corresponding 29 | Service resource and needs three main configuration parameters: 30 | 31 | - name of the Deployment 32 | - name of the Endpoints associated with the Service 33 | - target address:port to which requests should be proxied (typically 34 | the Service name and port) 35 | 36 | The ZPA then consists of another Deployment of the ZPA and another 37 | Service selecting the ZPA pods. Instead of connecting to the original 38 | Service, clients must connect to the ZPA Service resource. 39 | 40 | Finally, the ZPA needs a service account with sufficient permissions 41 | to watch Endpoints and scale Deployments. 42 | 43 | ``` yaml 44 | apiVersion: apps/v1 45 | kind: Deployment 46 | metadata: 47 | name: myzpa 48 | labels: 49 | app.kubernetes.io/name: myzpa 50 | spec: 51 | replicas: 2 52 | selector: 53 | matchLabels: 54 | app.kubernetes.io/name: myzpa 55 | template: 56 | metadata: 57 | labels: 58 | app.kubernetes.io/name: myzpa 59 | spec: 60 | serviceAccountName: zpa 61 | containers: 62 | - name: zpa 63 | image: greenkeytech/zero-pod-autoscaler:0.4.0 64 | imagePullPolicy: IfNotPresent 65 | args: 66 | - --namespace=$(NAMESPACE) 67 | - --address=0.0.0.0:80 68 | - --deployment=myapp 69 | - --endpoints=myapp 70 | - --target=myapp:80 71 | ports: 72 | - name: proxy 73 | protocol: TCP 74 | containerPort: 80 75 | 76 | --- 77 | 78 | apiVersion: v1 79 | kind: Service 80 | metadata: 81 | name: myzpa 82 | spec: 83 | type: ClusterIP 84 | ports: 85 | - name: http 86 | port: 80 87 | targetPort: proxy 88 | protocol: TCP 89 | selector: 90 | app.kubernetes.io/name: myzpa 91 | 92 | --- 93 | 94 | apiVersion: v1 95 | kind: ServiceAccount 96 | metadata: 97 | name: zpa 98 | 99 | --- 100 | 101 | kind: Role 102 | apiVersion: rbac.authorization.k8s.io/v1beta1 103 | metadata: 104 | name: read-deployments 105 | rules: 106 | - apiGroups: ["apps"] 107 | resources: ["deployments"] 108 | verbs: ["get", "list", "watch"] 109 | 110 | --- 111 | 112 | kind: Role 113 | apiVersion: rbac.authorization.k8s.io/v1beta1 114 | metadata: 115 | name: scale-deployments 116 | rules: 117 | - apiGroups: ["apps"] 118 | resources: ["deployments", "deployments/scale"] 119 | verbs: ["patch", "update"] 120 | 121 | --- 122 | 123 | kind: Role 124 | apiVersion: rbac.authorization.k8s.io/v1beta1 125 | metadata: 126 | name: read-endpoints 127 | rules: 128 | - apiGroups: [""] 129 | resources: ["endpoints"] 130 | verbs: ["get", "list", "watch"] 131 | 132 | --- 133 | 134 | kind: RoleBinding 135 | apiVersion: rbac.authorization.k8s.io/v1beta1 136 | metadata: 137 | name: zpa-read-endpoints 138 | subjects: 139 | - kind: ServiceAccount 140 | name: zpa 141 | roleRef: 142 | kind: Role 143 | name: read-endpoints 144 | apiGroup: rbac.authorization.k8s.io 145 | 146 | --- 147 | 148 | kind: RoleBinding 149 | apiVersion: rbac.authorization.k8s.io/v1beta1 150 | metadata: 151 | name: zpa-read-deployments 152 | subjects: 153 | - kind: ServiceAccount 154 | name: zpa 155 | roleRef: 156 | kind: Role 157 | name: read-deployments 158 | apiGroup: rbac.authorization.k8s.io 159 | 160 | --- 161 | 162 | kind: RoleBinding 163 | apiVersion: rbac.authorization.k8s.io/v1beta1 164 | metadata: 165 | name: zpa-scale-deployments 166 | subjects: 167 | - kind: ServiceAccount 168 | name: zpa 169 | roleRef: 170 | kind: Role 171 | name: scale-deployments 172 | apiGroup: rbac.authorization.k8s.io 173 | ``` 174 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GreenKey Technologies 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you 4 | // may not use this file except in compliance with the License. You 5 | // may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | package main 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | "log" 20 | "net" 21 | "net/http" 22 | "os" 23 | "os/signal" 24 | "sync" 25 | "sync/atomic" 26 | "syscall" 27 | "time" 28 | 29 | // required to work with clientcmd/api 30 | flag "github.com/spf13/pflag" 31 | 32 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 | 34 | "github.com/greenkeytech/zero-pod-autoscaler/pkg/kubeconfig" 35 | "github.com/greenkeytech/zero-pod-autoscaler/pkg/proxy" 36 | "github.com/greenkeytech/zero-pod-autoscaler/pkg/scaler" 37 | ) 38 | 39 | // global health status 40 | var globalHealth = int32(1) 41 | 42 | func Healthy() bool { 43 | return atomic.LoadInt32(&globalHealth) != 0 44 | } 45 | 46 | func SetUnhealthy() { 47 | if atomic.CompareAndSwapInt32(&globalHealth, 1, 0) { 48 | log.Printf("system is unhealthy") 49 | } 50 | } 51 | 52 | func SetHealthy() { 53 | if atomic.CompareAndSwapInt32(&globalHealth, 0, 1) { 54 | log.Printf("system is healthy") 55 | } 56 | } 57 | 58 | type acceptResult struct { 59 | conn net.Conn 60 | err error 61 | } 62 | 63 | func Iterate(ctx context.Context, accepts chan acceptResult, wg sync.WaitGroup, target string, sc *scaler.Scaler) error { 64 | select { 65 | case <-ctx.Done(): 66 | return ctx.Err() 67 | case a := <-accepts: 68 | conn, err := a.conn, a.err 69 | if err != nil { 70 | return fmt.Errorf("failed to accept connection: %w", err) 71 | } 72 | 73 | log.Printf("%s->%s: accept connection", conn.RemoteAddr(), conn.LocalAddr()) 74 | 75 | wg.Add(1) 76 | go func() { 77 | start := time.Now() 78 | 79 | defer wg.Done() 80 | defer conn.Close() 81 | 82 | err := sc.UseConnection(func() error { 83 | // race condition here: could become 84 | // unavailable immediately after 85 | // reporting available, but not much 86 | // we can do 87 | 88 | select { 89 | case <-sc.Available(): 90 | return proxy.ProxyTo(conn, target) 91 | case <-time.After(0): 92 | // was not immediately available; continue below 93 | } 94 | 95 | log.Printf("%s->%s: waiting for upstream to become available", 96 | conn.RemoteAddr(), conn.LocalAddr()) 97 | select { 98 | case <-sc.Available(): 99 | log.Printf("%s->%s: upstream available after %s", 100 | conn.RemoteAddr(), conn.LocalAddr(), 101 | time.Since(start)) 102 | return proxy.ProxyTo(conn, target) 103 | case <-time.After(5 * time.Minute): 104 | return fmt.Errorf("timed out waiting for available upstream") 105 | } 106 | }) 107 | if err != nil { 108 | log.Printf("%s->%s: failed to proxy: %v", conn.RemoteAddr(), conn.LocalAddr(), err) 109 | SetUnhealthy() 110 | } else { 111 | SetHealthy() 112 | } 113 | 114 | log.Printf("%s->%s: close connection after %s", 115 | conn.RemoteAddr(), conn.LocalAddr(), time.Since(start)) 116 | }() 117 | 118 | return nil 119 | } 120 | } 121 | 122 | func ListenAndProxy(ctx context.Context, addr, target string, sc *scaler.Scaler) error { 123 | ln, err := net.Listen("tcp", addr) 124 | if err != nil { 125 | return fmt.Errorf("failed to listen on `%s`: %v", err) 126 | } 127 | log.Printf("proxy listening on %s", addr) 128 | 129 | accepts := make(chan acceptResult) 130 | stop := make(chan struct{}) 131 | go func() { 132 | for { 133 | select { 134 | case <-stop: 135 | return 136 | case <-time.After(0): 137 | // proceed 138 | } 139 | 140 | conn, err := ln.Accept() 141 | accepts <- acceptResult{conn, err} 142 | } 143 | }() 144 | 145 | wg := sync.WaitGroup{} 146 | 147 | for { 148 | err := Iterate(ctx, accepts, wg, target, sc) 149 | if err != nil { 150 | log.Printf("refusing new connections") 151 | ln.Close() 152 | close(stop) 153 | break 154 | } 155 | } 156 | 157 | log.Printf("draining existing connections") 158 | wg.Wait() 159 | return nil 160 | } 161 | 162 | func makeListOptions(name, labelSelector string) metav1.ListOptions { 163 | options := metav1.ListOptions{} 164 | if name != "" { 165 | options.FieldSelector = fmt.Sprintf("metadata.name=%s", name) 166 | } 167 | options.LabelSelector = labelSelector 168 | return options 169 | } 170 | 171 | func initializeSignalHandlers(cancel func()) { 172 | c := make(chan os.Signal, 1) 173 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 174 | go func() { 175 | log.Printf("received signal: %s", <-c) 176 | cancel() 177 | }() 178 | } 179 | 180 | func runAdminServer(ctx context.Context, addr string) error { 181 | ln, err := net.Listen("tcp", addr) 182 | if err != nil { 183 | return fmt.Errorf("failed to listen on `%s`: %v", addr, err) 184 | } 185 | 186 | log.Printf("admin server listening on %s", addr) 187 | 188 | mux := http.NewServeMux() 189 | mux.HandleFunc("/healthz", func(resp http.ResponseWriter, req *http.Request) { 190 | if !Healthy() { 191 | resp.WriteHeader(http.StatusServiceUnavailable) 192 | return 193 | } 194 | resp.WriteHeader(http.StatusOK) 195 | }) 196 | 197 | s := &http.Server{ 198 | Addr: addr, 199 | Handler: mux, 200 | ReadTimeout: 5 * time.Second, 201 | WriteTimeout: 5 * time.Second, 202 | } 203 | go func() { 204 | if err := s.Serve(ln); err != nil { 205 | log.Printf("admin server exited: %v", err) 206 | } 207 | }() 208 | 209 | <-ctx.Done() 210 | 211 | shutdownCtx, _ := context.WithTimeout(context.Background(), 10*time.Second) 212 | return s.Shutdown(shutdownCtx) 213 | } 214 | 215 | func main() { 216 | flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 217 | loadingRules, configOverrides := kubeconfig.BindKubeFlags(flags) 218 | 219 | addr := flags.String("address", "localhost:3000", "listen address") 220 | adminAddr := flags.String("admin-address", "localhost:8080", "listen address of http admin interface") 221 | deployment := flags.String("deployment", "", "name of Deployment to scale") 222 | labelSelector := flags.StringP("selector", "l", "", "label selector of Deployment") 223 | ep := flags.String("endpoints", "", "name of Endpoints to watch for ready addresses") 224 | target := flags.String("target", "", "target address to which to proxy requests") 225 | ttl := flags.Duration("ttl", 1*time.Hour, "idle duration before scaling to zero") 226 | 227 | if err := flags.Parse(os.Args[1:]); err != nil { 228 | log.Fatal(err) 229 | } 230 | 231 | // the kube flags bound above include a --namespace flag... 232 | namespace := "default" 233 | if configOverrides.Context.Namespace != "" { 234 | namespace = configOverrides.Context.Namespace 235 | } 236 | 237 | clientset, err := kubeconfig.BuildClientset(loadingRules, configOverrides) 238 | if err != nil { 239 | log.Fatalf("error building kubernetes clientset: %v", err) 240 | } 241 | 242 | ctx, cancel := context.WithCancel(context.Background()) 243 | initializeSignalHandlers(cancel) 244 | 245 | wg := sync.WaitGroup{} 246 | 247 | //////////////////////////////////////////////////////////// 248 | // admin server 249 | wg.Add(1) 250 | go func() { 251 | defer wg.Done() 252 | err := runAdminServer(ctx, *adminAddr) 253 | if err != nil { 254 | log.Printf("admin server exited: %v") 255 | } 256 | }() 257 | 258 | //////////////////////////////////////////////////////////// 259 | // scaler 260 | sc, err := scaler.New(context.Background(), clientset, namespace, 261 | makeListOptions(*deployment, *labelSelector), 262 | makeListOptions(*ep, *labelSelector), 263 | *target, 264 | *ttl) 265 | if err != nil { 266 | log.Fatal(err) 267 | } 268 | go func() { 269 | err := sc.Run(context.Background()) 270 | if err != nil { 271 | log.Printf("scaler exited: %v", err) 272 | } 273 | }() 274 | 275 | //////////////////////////////////////////////////////////// 276 | // proxy server 277 | wg.Add(1) 278 | go func() { 279 | defer wg.Done() 280 | err := ListenAndProxy(ctx, *addr, *target, sc) 281 | if err != nil { 282 | log.Printf("proxy exited: %v", err) 283 | } 284 | }() 285 | 286 | wg.Wait() 287 | log.Printf("exiting") 288 | } 289 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /pkg/scaler/scaler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GreenKey Technologies 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you 4 | // may not use this file except in compliance with the License. You 5 | // may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | package scaler 15 | 16 | import ( 17 | "context" 18 | "encoding/json" 19 | "fmt" 20 | "log" 21 | "net" 22 | "time" 23 | 24 | appsv1 "k8s.io/api/apps/v1" 25 | autoscalingv1 "k8s.io/api/autoscaling/v1" 26 | corev1 "k8s.io/api/core/v1" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | types "k8s.io/apimachinery/pkg/types" 29 | "k8s.io/client-go/informers" 30 | "k8s.io/client-go/kubernetes" 31 | "k8s.io/client-go/tools/cache" 32 | ) 33 | 34 | var KeyScaleDownAt = "zero-pod-autoscaler/scale-down-at" 35 | 36 | type Scaler struct { 37 | Client kubernetes.Interface 38 | Namespace string 39 | Name string 40 | 41 | // Target address to which requests are proxied. We need this 42 | // because endpoints reporting available doesn't mean the 43 | // Service can actually handle a request... so we ping the 44 | // actual target also. 45 | Target string 46 | 47 | TTL time.Duration 48 | availableRequest chan chan chan struct{} 49 | scaleUp chan int 50 | connectionInc chan int 51 | 52 | updated chan interface{} 53 | deleted chan interface{} 54 | } 55 | 56 | func New( 57 | ctx context.Context, 58 | client kubernetes.Interface, 59 | namespace string, 60 | deployOptions, epOptions metav1.ListOptions, 61 | target string, 62 | ttl time.Duration, 63 | ) (*Scaler, error) { 64 | var deploy appsv1.Deployment 65 | var ep corev1.Endpoints 66 | 67 | if list, err := client.AppsV1().Deployments(namespace).List(deployOptions); err != nil { 68 | return nil, err 69 | } else { 70 | if len(list.Items) > 1 { 71 | return nil, fmt.Errorf("matched %d Deployments", len(list.Items)) 72 | } 73 | 74 | if len(list.Items) == 0 { 75 | return nil, fmt.Errorf("did not match any Deployments") 76 | } 77 | 78 | deploy = list.Items[0] 79 | } 80 | 81 | if list, err := client.CoreV1().Endpoints(namespace).List(epOptions); err != nil { 82 | if err != nil { 83 | return nil, err 84 | } 85 | } else { 86 | if len(list.Items) > 1 { 87 | return nil, fmt.Errorf("matched %d Endpoints", len(list.Items)) 88 | } 89 | 90 | if len(list.Items) == 0 { 91 | return nil, fmt.Errorf("did not match any Endpoints") 92 | } 93 | 94 | ep = list.Items[0] 95 | } 96 | 97 | log.Printf("watching %s/%s", "Endpoints", ep.Name) 98 | log.Printf("watching %s/%s", "Deployment", deploy.Name) 99 | 100 | fieldSelector := fmt.Sprintf("metadata.name=%s", deploy.Name) 101 | 102 | factory := informers.NewSharedInformerFactoryWithOptions( 103 | client, 104 | 1*time.Minute, 105 | informers.WithNamespace(namespace), 106 | informers.WithTweakListOptions(func(opts *metav1.ListOptions) { 107 | opts.FieldSelector = fieldSelector 108 | })) 109 | 110 | updated := make(chan interface{}) 111 | deleted := make(chan interface{}) 112 | availableRequest := make(chan chan chan struct{}) 113 | scaleUp := make(chan int) 114 | connectionInc := make(chan int) 115 | 116 | funcs := cache.ResourceEventHandlerFuncs{ 117 | AddFunc: func(obj interface{}) { 118 | updated <- obj 119 | }, 120 | UpdateFunc: func(oldObj, obj interface{}) { 121 | updated <- obj 122 | }, 123 | DeleteFunc: func(obj interface{}) { 124 | deleted <- obj 125 | }, 126 | } 127 | 128 | if informer := factory.Apps().V1().Deployments().Informer(); true { 129 | informer.AddEventHandler(funcs) 130 | go informer.Run(ctx.Done()) 131 | } 132 | 133 | if informer := factory.Core().V1().Endpoints().Informer(); true { 134 | informer.AddEventHandler(funcs) 135 | go informer.Run(ctx.Done()) 136 | } 137 | 138 | sc := &Scaler{ 139 | Client: client, 140 | Namespace: namespace, 141 | Name: deploy.Name, 142 | Target: target, 143 | TTL: ttl, 144 | availableRequest: availableRequest, 145 | scaleUp: scaleUp, 146 | connectionInc: connectionInc, 147 | updated: updated, 148 | deleted: deleted, 149 | } 150 | 151 | return sc, nil 152 | } 153 | 154 | func (sc *Scaler) TryConnect(ctx context.Context) error { 155 | dialer := net.Dialer{} 156 | timeout_ms := 10.0 157 | factor := 5.0 // timeout series: 10, 50, 250, 1250, 6250, 31250 158 | for { 159 | if err := ctx.Err(); err != nil { 160 | return err 161 | } 162 | 163 | subctx, _ := context.WithTimeout(ctx, time.Duration(timeout_ms)*time.Millisecond) 164 | timeout_ms *= factor 165 | conn, err := dialer.DialContext(subctx, "tcp", sc.Target) 166 | if err != nil { 167 | log.Printf("failed test connection to %s: %v", sc.Target, err) 168 | continue 169 | } 170 | 171 | conn.Close() 172 | return nil 173 | } 174 | } 175 | 176 | func (sc *Scaler) Run(ctx context.Context) error { 177 | replicas := int32(-1) 178 | readyAddresses := -1 179 | notReadyAddresses := -1 180 | connCount := 0 181 | 182 | // channel is closed when upstream is available 183 | var available chan struct{} 184 | closedChan := make(chan struct{}) 185 | close(closedChan) 186 | 187 | resourceVersion := "" 188 | 189 | scaleDownAt := time.Now().Add(sc.TTL) 190 | 191 | for { 192 | select { 193 | case <-ctx.Done(): 194 | return fmt.Errorf("%v", ctx.Err()) 195 | case i := <-sc.connectionInc: 196 | connCount += i 197 | case obj := <-sc.updated: 198 | switch resource := obj.(type) { 199 | case *corev1.Endpoints: 200 | r := 0 201 | nr := 0 202 | 203 | for _, subset := range resource.Subsets { 204 | r += len(subset.Addresses) 205 | nr += len(subset.NotReadyAddresses) 206 | } 207 | 208 | if r != readyAddresses || nr != notReadyAddresses { 209 | log.Printf("%s/%s: readyAddresses=%d notReadyAddresses=%d", 210 | "Endpoints", resource.Name, r, nr) 211 | } 212 | 213 | readyAddresses, notReadyAddresses = r, nr 214 | 215 | if readyAddresses == 0 { 216 | continue 217 | } 218 | 219 | // nothing is waiting 220 | if available == nil { 221 | continue 222 | } 223 | 224 | log.Printf("%s/%s has ready addresses; confirming can connect to %s", 225 | "Endpoints", resource.Name, sc.Target) 226 | 227 | subctx, _ := context.WithTimeout(ctx, 15*time.Second) 228 | if err := sc.TryConnect(subctx); err != nil { 229 | log.Fatalf("%s/%s has ready addresses but failed to connect to %s: %v", 230 | "Endpoints", resource.Name, sc.Target, err) 231 | } 232 | 233 | log.Printf("%s available; notifying waiters", resource.Name) 234 | close(available) 235 | available = nil 236 | case *appsv1.Deployment: 237 | resourceVersion = resource.ResourceVersion 238 | 239 | if timestamp, ok := resource.Annotations[KeyScaleDownAt]; ok { 240 | if t, err := time.Parse(time.RFC3339, timestamp); err == nil { 241 | scaleDownAt = t 242 | } 243 | } 244 | 245 | if resource.Spec.Replicas != nil { 246 | if replicas != *resource.Spec.Replicas { 247 | log.Printf("%s/%s: replicas: %d", 248 | "Deployment", resource.Name, *resource.Spec.Replicas) 249 | } 250 | replicas = *resource.Spec.Replicas 251 | } 252 | } 253 | case obj := <-sc.deleted: 254 | switch resource := obj.(type) { 255 | case *corev1.Endpoints: 256 | log.Fatalf("%s/%s: deleted", "Endpoints", resource.Name) 257 | case *appsv1.Deployment: 258 | log.Fatalf("%s/%s: deleted", "Deployment", resource.Name) 259 | } 260 | case reply := <-sc.availableRequest: 261 | // set time to scale down 262 | sc.extendScaleDownAtMaybe(scaleDownAt) 263 | 264 | if readyAddresses > 0 { 265 | // is currently available; send the already-closed channel 266 | reply <- closedChan 267 | continue 268 | } 269 | 270 | // nothing ready, reply with channel that gets closed when ready 271 | if available == nil { 272 | available = make(chan struct{}) 273 | } 274 | reply <- available 275 | 276 | if replicas == 0 { 277 | go func() { sc.scaleUp <- 0 }() 278 | } 279 | case attemptNumber := <-sc.scaleUp: 280 | if replicas == 0 { 281 | if err := sc.updateScale(resourceVersion, 1); err != nil { 282 | log.Printf("%s/%s: failed to scale up: %v %T", 283 | "Deployment", sc.Name, err, err) 284 | // try again; usual error is that resource is out of date 285 | // TODO: try again ONLY when error is that resource is out of date 286 | if attemptNumber < 10 { 287 | go func() { sc.scaleUp <- attemptNumber + 1 }() 288 | } 289 | } 290 | } 291 | case <-time.After(1 * time.Second): 292 | if connCount > 0 { 293 | sc.extendScaleDownAtMaybe(scaleDownAt) 294 | } 295 | 296 | if connCount == 0 && replicas > 0 && time.Now().After(scaleDownAt) { 297 | log.Printf("%s/%s: scaling down after %s: replicas=%d connections=%d", 298 | "Deployment", sc.Name, sc.TTL, replicas, connCount) 299 | 300 | if err := sc.updateScale(resourceVersion, 0); err != nil { 301 | log.Printf("%s/%s: failed to scale to zero: %v", 302 | "Deployment", sc.Name, err) 303 | } 304 | } 305 | } 306 | } 307 | } 308 | 309 | func (sc *Scaler) extendScaleDownAtMaybe(scaleDownAt time.Time) { 310 | if !time.Now().After(scaleDownAt.Add(sc.TTL / -2)) { 311 | return 312 | } 313 | 314 | path := fmt.Sprintf("/metadata/annotations/%s", JsonPatchEscape(KeyScaleDownAt)) 315 | 316 | patch := []map[string]string{ 317 | { 318 | "op": "replace", 319 | "path": path, 320 | "value": time.Now().Add(sc.TTL).Format(time.RFC3339), 321 | }, 322 | } 323 | 324 | body, err := json.Marshal(patch) 325 | if err != nil { 326 | log.Printf("failed to marshal patch to json: %v", err) 327 | } 328 | 329 | if _, err := sc.Client.AppsV1().Deployments(sc.Namespace). 330 | Patch(sc.Name, types.JSONPatchType, body); err != nil { 331 | log.Printf("%s/%s: failed to patch: %v", 332 | "Deployment", sc.Name, err) 333 | } 334 | 335 | log.Printf("%s/%s: updated scaleDownAt to %s from now", 336 | "Deployment", sc.Name, sc.TTL) 337 | } 338 | 339 | func (sc *Scaler) updateScale(resourceVersion string, replicas int32) error { 340 | deployments := sc.Client.AppsV1().Deployments(sc.Namespace) 341 | 342 | scale := autoscalingv1.Scale{} 343 | scale.Namespace = sc.Namespace 344 | scale.Name = sc.Name 345 | scale.ResourceVersion = resourceVersion 346 | 347 | scale.Spec.Replicas = replicas 348 | 349 | if _, err := deployments.UpdateScale(sc.Name, &scale); err != nil { 350 | return err 351 | } 352 | 353 | log.Printf("%s/%s: scaled to %d", "Deployment", sc.Name, replicas) 354 | 355 | return nil 356 | } 357 | 358 | func (sc *Scaler) UseConnection(f func() error) error { 359 | sc.connectionInc <- 1 360 | err := f() 361 | sc.connectionInc <- -1 362 | return err 363 | } 364 | 365 | // Available returns a channel that will be closed when upstream is 366 | // available. The returned channel may already be closed if upstream 367 | // is currently available. 368 | func (sc *Scaler) Available() (available chan struct{}) { 369 | reply := make(chan chan struct{}) 370 | sc.availableRequest <- reply 371 | return <-reply 372 | } 373 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= 4 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 5 | github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= 6 | github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= 7 | github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= 8 | github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 9 | github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 10 | github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= 11 | github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= 12 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 13 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 14 | github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 15 | github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 16 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 17 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 20 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 22 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= 23 | github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 24 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 25 | github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 26 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 27 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 28 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 29 | github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= 30 | github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= 31 | github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= 32 | github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= 33 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= 34 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 35 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 36 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 37 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= 38 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 39 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 40 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 41 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 42 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 43 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 44 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 45 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 46 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 47 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 48 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 49 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 50 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 51 | github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= 52 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 53 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 54 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 55 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 56 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 57 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 58 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= 59 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 60 | github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= 61 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 62 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 63 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 64 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 65 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 66 | github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= 67 | github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 68 | github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 69 | github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= 70 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 71 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 72 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 73 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 74 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 75 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 76 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 77 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 78 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 79 | github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 80 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 81 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 82 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 83 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 84 | github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 85 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 86 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 87 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 88 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 89 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 90 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 91 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 92 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 93 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 94 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 95 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 96 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 97 | github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 98 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 99 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 100 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 101 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 102 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 103 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 104 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 105 | github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 106 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 107 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 108 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 109 | go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= 110 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 111 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 112 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 113 | golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= 114 | golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 115 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 116 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 117 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 118 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 119 | golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 120 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 121 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 122 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 123 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 124 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 125 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 126 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 127 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= 128 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 129 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 130 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 131 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 132 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 133 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 134 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 135 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 136 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 137 | golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 138 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 139 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 140 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 141 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 142 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 143 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= 144 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 145 | golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 146 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 147 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 148 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 149 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 150 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 151 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= 152 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 153 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 154 | golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 155 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 156 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 157 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 158 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 159 | google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= 160 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 161 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 162 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 163 | google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= 164 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 165 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 166 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 167 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE= 168 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 169 | google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= 170 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 171 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 172 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 173 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 174 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 175 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 176 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 177 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 178 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 179 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 180 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 181 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 182 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 183 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 184 | k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= 185 | k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= 186 | k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= 187 | k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= 188 | k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg= 189 | k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= 190 | k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 191 | k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 192 | k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 193 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 194 | k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 195 | k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= 196 | k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= 197 | k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= 198 | sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= 199 | sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= 200 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 201 | --------------------------------------------------------------------------------