├── .gitignore ├── tools.go ├── .gometalinter.json ├── .github └── workflows │ └── ci.yml ├── .editorconfig ├── go.mod ├── apis └── condition │ └── v1 │ ├── zz_generated.deepcopy.go │ ├── util.go │ └── types.go ├── Makefile ├── process ├── http.go ├── work_queue.go ├── generic_worker.go └── generic.go ├── logz ├── crash.go └── logz.go ├── options ├── controller.go ├── logger.go ├── client.go └── leader_election.go ├── README.md ├── handlers ├── generic_handler.go ├── lookup_handler.go └── controlled_resource_handler.go ├── flagutil ├── flags_test.go └── flags.go ├── app ├── aux_server.go └── app.go ├── types.go ├── LICENSE └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | /.idea/ 3 | /.vscode/ 4 | /build/bin/ 5 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package ctrl 4 | 5 | // https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module 6 | 7 | import ( 8 | _ "k8s.io/code-generator/cmd/deepcopy-gen" 9 | ) 10 | -------------------------------------------------------------------------------- /.gometalinter.json: -------------------------------------------------------------------------------- 1 | { 2 | "Concurrency": 4, 3 | "Cyclo": 15, 4 | "Deadline": "5m", 5 | "Disable": [ 6 | "interfacer" 7 | ], 8 | "Exclude": [ 9 | "comment", 10 | "zz_generated." 11 | ], 12 | "Test": true, 13 | "Vendor": true 14 | } -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.12.7 14 | 15 | - name: Check out code 16 | uses: actions/checkout@v1 17 | 18 | - name: Test 19 | env: 20 | GO111MODULE: on 21 | run: make test 22 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | [**] 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | 11 | [**.go] 12 | insert_final_newline = true 13 | indent_style = tab 14 | 15 | [Makefile] 16 | insert_final_newline = true 17 | indent_style = tab 18 | 19 | [**.{yml,yaml,md}] 20 | insert_final_newline = true 21 | indent_style = space 22 | indent_size = 2 23 | 24 | [BUILD.bazel] 25 | insert_final_newline = true 26 | indent_style = space 27 | indent_size = 4 28 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/atlassian/ctrl 2 | 3 | go 1.13.1 4 | 5 | require ( 6 | github.com/ash2k/stager v0.0.0-20170622123058-6e9c7b0eacd4 7 | github.com/go-chi/chi v4.0.2+incompatible 8 | github.com/pkg/errors v0.8.0 9 | github.com/prometheus/client_golang v0.9.2 10 | github.com/stretchr/testify v1.3.0 11 | go.uber.org/atomic v1.4.0 // indirect 12 | go.uber.org/multierr v1.1.0 // indirect 13 | go.uber.org/zap v1.10.0 14 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 15 | k8s.io/api v0.0.0-20191003000013-35e20aa79eb8 16 | k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655 17 | k8s.io/client-go v0.0.0-20191003000419-f68efa97b39e 18 | k8s.io/code-generator v0.0.0-20190927045949-f81bca4f5e85 19 | ) 20 | -------------------------------------------------------------------------------- /apis/condition/v1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Generated file, do not modify manually! 4 | 5 | // Code generated by deepcopy-gen. DO NOT EDIT. 6 | 7 | package v1 8 | 9 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 10 | func (in *Condition) DeepCopyInto(out *Condition) { 11 | *out = *in 12 | in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) 13 | return 14 | } 15 | 16 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. 17 | func (in *Condition) DeepCopy() *Condition { 18 | if in == nil { 19 | return nil 20 | } 21 | out := new(Condition) 22 | in.DeepCopyInto(out) 23 | return out 24 | } 25 | -------------------------------------------------------------------------------- /apis/condition/v1/util.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | func FindCondition(conditions []Condition, conditionType ConditionType) (int /* index */, *Condition) { 4 | for i, condition := range conditions { 5 | if condition.Type == conditionType { 6 | return i, &condition 7 | } 8 | } 9 | return -1, nil 10 | } 11 | 12 | func CheckIfConditionChanged(currentCond, newCond *Condition) bool { 13 | return currentCond == nil || 14 | currentCond.Status != newCond.Status || 15 | currentCond.Reason != newCond.Reason || 16 | currentCond.Message != newCond.Message 17 | } 18 | 19 | func PrepareCondition(currentConditions []Condition, newCondition *Condition) bool { 20 | needsUpdate := true 21 | 22 | // Try to find resource condition 23 | _, oldCondition := FindCondition(currentConditions, newCondition.Type) 24 | 25 | if oldCondition != nil { 26 | // We are updating an existing condition, so we need to check if it has changed. 27 | if newCondition.Status == oldCondition.Status { 28 | newCondition.LastTransitionTime = oldCondition.LastTransitionTime 29 | } 30 | 31 | needsUpdate = CheckIfConditionChanged(oldCondition, newCondition) 32 | } 33 | return needsUpdate 34 | } 35 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ALL_GO_FILES=$$(find . -type f -name '*.go' -not -path "./vendor/*") 2 | KUBE="kubernetes-1.16.1" 3 | 4 | .PHONY: fmt 5 | fmt: 6 | go build -o build/bin/goimports golang.org/x/tools/cmd/goimports 7 | build/bin/goimports -w=true -d $(ALL_GO_FILES) 8 | 9 | .PHONY: test 10 | test: 11 | go test -race ./... 12 | 13 | # to update kubernetes version: 14 | # 1. copy https://github.com/kubernetes/kubernetes/blob/v1.15.1/go.mod "require" section into go.mod 15 | # 2. remove all v0.0.0 k8s.io/* statements 16 | # 3. run this command 17 | .PHONY: update-kube 18 | update-kube: 19 | go get \ 20 | k8s.io/api/core/v1@$(KUBE) \ 21 | k8s.io/apimachinery@$(KUBE) \ 22 | k8s.io/client-go@$(KUBE) \ 23 | k8s.io/code-generator/cmd/deepcopy-gen@$(KUBE) \ 24 | github.com/stretchr/testify@v1.3.0 25 | go mod tidy 26 | 27 | .PHONY: generate-deepcopy 28 | generate-deepcopy: 29 | go build -o build/bin/deepcopy-gen k8s.io/code-generator/cmd/deepcopy-gen 30 | build/bin/deepcopy-gen \ 31 | --v 1 --logtostderr \ 32 | --go-header-file "build/boilerplate.go.txt" \ 33 | --input-dirs "./apis/condition/v1" \ 34 | --output-base "$(CURDIR)" \ 35 | --bounding-dirs "./apis/condition/v1" \ 36 | --output-file-base zz_generated.deepcopy 37 | -------------------------------------------------------------------------------- /process/http.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "sync" 7 | "time" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func StartStopServer(ctx context.Context, srv *http.Server, shutdownTimeout time.Duration) error { 13 | return StartStopTLSServer(ctx, srv, shutdownTimeout, "", "") 14 | } 15 | 16 | func StartStopTLSServer(ctx context.Context, srv *http.Server, shutdownTimeout time.Duration, certFile, keyFile string) error { 17 | var wg sync.WaitGroup 18 | defer wg.Wait() // wait for goroutine to shutdown active connections 19 | ctx, cancel := context.WithCancel(ctx) 20 | defer cancel() 21 | wg.Add(1) 22 | go func() { 23 | defer wg.Done() 24 | <-ctx.Done() 25 | c, cancel := context.WithTimeout(context.Background(), shutdownTimeout) 26 | defer cancel() 27 | if srv.Shutdown(c) != nil { 28 | srv.Close() // nolint: errcheck,gas,gosec 29 | // unhandled error above, but we are terminating anyway 30 | } 31 | }() 32 | 33 | var err error 34 | if certFile == "" || keyFile == "" { 35 | err = srv.ListenAndServe() 36 | } else { 37 | err = srv.ListenAndServeTLS(certFile, keyFile) 38 | } 39 | 40 | if err != http.ErrServerClosed { 41 | // Failed to start or dirty shutdown 42 | return errors.WithStack(err) 43 | } 44 | // Clean shutdown 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /logz/crash.go: -------------------------------------------------------------------------------- 1 | package logz 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "os" 8 | "runtime/debug" 9 | "time" 10 | ) 11 | 12 | func LogStructuredPanic() { 13 | if r := recover(); r != nil { 14 | logStructuredPanic(os.Stderr, r, time.Now(), debug.Stack()) 15 | panic(r) 16 | } 17 | } 18 | 19 | // internally zap is overly opinionated about what to do when the log level is fatal or panic 20 | // it chooses to call os.exit or panic if the level is set. There does not appear to be a simple 21 | // way to work around that choice so we build the log message by hand instead. 22 | func logStructuredPanic(out io.Writer, panicValue interface{}, now time.Time, stack []byte) { 23 | bytes, err := json.Marshal(struct { 24 | Level string `json:"level"` 25 | Time string `json:"time"` 26 | Message string `json:"msg"` 27 | Stack string `json:"stack"` 28 | }{ 29 | Level: "fatal", 30 | Time: now.Format(time.RFC3339), 31 | Message: fmt.Sprintf("%v", panicValue), 32 | Stack: string(stack), 33 | }) 34 | if err != nil { 35 | fmt.Fprintf(out, "error while serializing panic: %+v\n", err) // nolint: errcheck, gas 36 | fmt.Fprintf(out, "original panic: %+v\n", panicValue) // nolint: errcheck, gas 37 | return 38 | } 39 | fmt.Fprintf(out, "%s\n", bytes) // nolint: errcheck, gas 40 | } 41 | -------------------------------------------------------------------------------- /options/controller.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/atlassian/ctrl" 7 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | ) 9 | 10 | const ( 11 | DefaultResyncPeriod = 20 * time.Minute 12 | DefaultWorkers = 2 13 | ) 14 | 15 | type GenericControllerOptions struct { 16 | ResyncPeriod time.Duration 17 | Workers uint 18 | } 19 | 20 | func (o *GenericControllerOptions) DefaultAndValidate() []error { 21 | var allErrors []error 22 | if o.ResyncPeriod == 0 { 23 | o.ResyncPeriod = DefaultResyncPeriod 24 | } 25 | if o.Workers == 0 { 26 | o.Workers = DefaultWorkers 27 | } 28 | return allErrors 29 | } 30 | 31 | func BindGenericControllerFlags(o *GenericControllerOptions, fs ctrl.FlagSet) { 32 | fs.DurationVar(&o.ResyncPeriod, "resync-period", DefaultResyncPeriod, "Resync period for informers") 33 | fs.UintVar(&o.Workers, "workers", DefaultWorkers, "Number of workers that handle events from informers") 34 | } 35 | 36 | type GenericNamespacedControllerOptions struct { 37 | GenericControllerOptions 38 | Namespace string 39 | } 40 | 41 | func BindGenericNamespacedControllerFlags(o *GenericNamespacedControllerOptions, fs ctrl.FlagSet) { 42 | BindGenericControllerFlags(&o.GenericControllerOptions, fs) 43 | fs.StringVar(&o.Namespace, "namespace", meta_v1.NamespaceAll, "Namespace to use. All namespaces are used if empty string or omitted") 44 | } 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ctrl 2 | 3 | A tiny framework for building a particular type of Kubernetes controllers. Opinionated, of course. 4 | 5 | ## Contributing 6 | 7 | Pull requests, issues and comments welcome. For pull requests: 8 | 9 | * Add tests for new features and bug fixes 10 | * Follow the existing style 11 | * Separate unrelated changes into multiple pull requests 12 | 13 | See the existing issues for things to start contributing. 14 | 15 | For bigger changes, make sure you start a discussion first by creating an issue and explaining the intended change. 16 | 17 | Atlassian requires contributors to sign a Contributor License Agreement, known as a CLA. This serves as a record 18 | stating that the contributor is entitled to contribute the code/documentation/translation to the project and is willing 19 | to have it used in distributions and derivative works (or is willing to transfer ownership). 20 | 21 | Prior to accepting your contributions we ask that you please follow the appropriate link below to digitally sign the 22 | CLA. The Corporate CLA is for those who are contributing as a member of an organization and the individual CLA is for 23 | those contributing as an individual. 24 | 25 | * [CLA for corporate contributors](https://na2.docusign.net/Member/PowerFormSigning.aspx?PowerFormId=e1c17c66-ca4d-4aab-a953-2c231af4a20b) 26 | * [CLA for individuals](https://na2.docusign.net/Member/PowerFormSigning.aspx?PowerFormId=3f94fbdc-2fbe-46ac-b14c-5d152700ae5d) 27 | 28 | ## License 29 | 30 | Copyright (c) 2018 Atlassian and others. Apache 2.0 licensed, see LICENSE file. 31 | -------------------------------------------------------------------------------- /options/logger.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/atlassian/ctrl" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | ) 10 | 11 | type LoggerOptions struct { 12 | LogLevel string 13 | LogEncoding string 14 | } 15 | 16 | func BindLoggerFlags(o *LoggerOptions, fs ctrl.FlagSet) { 17 | fs.StringVar(&o.LogLevel, "log-level", "info", `Sets the logger's output level.`) 18 | fs.StringVar(&o.LogEncoding, "log-encoding", "json", `Sets the logger's encoding. Valid values are "json" and "console".`) 19 | } 20 | 21 | func Logger(level zapcore.Level, encoder func(zapcore.EncoderConfig) zapcore.Encoder) *zap.Logger { 22 | cfg := zap.NewProductionEncoderConfig() 23 | cfg.EncodeTime = zapcore.ISO8601TimeEncoder 24 | cfg.TimeKey = "time" 25 | lockedSyncer := zapcore.Lock(zapcore.AddSync(os.Stderr)) 26 | return zap.New( 27 | zapcore.NewCore( 28 | encoder(cfg), 29 | lockedSyncer, 30 | level, 31 | ), 32 | zap.ErrorOutput(lockedSyncer), 33 | ) 34 | } 35 | 36 | func LoggerFromOptions(o LoggerOptions) *zap.Logger { 37 | var levelEnabler zapcore.Level 38 | switch o.LogLevel { 39 | case "debug": 40 | levelEnabler = zap.DebugLevel 41 | case "warn": 42 | levelEnabler = zap.WarnLevel 43 | case "error": 44 | levelEnabler = zap.ErrorLevel 45 | default: 46 | levelEnabler = zap.InfoLevel 47 | } 48 | var logEncoder func(zapcore.EncoderConfig) zapcore.Encoder 49 | if o.LogEncoding == "console" { 50 | logEncoder = zapcore.NewConsoleEncoder 51 | } else { 52 | logEncoder = zapcore.NewJSONEncoder 53 | } 54 | return Logger(levelEnabler, logEncoder) 55 | } 56 | -------------------------------------------------------------------------------- /handlers/generic_handler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/atlassian/ctrl" 5 | "github.com/atlassian/ctrl/logz" 6 | "go.uber.org/zap" 7 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/runtime/schema" 9 | "k8s.io/client-go/tools/cache" 10 | ) 11 | 12 | // This handler assumes that the Logger already has the obj_gk/ctrl_gk field set. 13 | type GenericHandler struct { 14 | Logger *zap.Logger 15 | WorkQueue ctrl.WorkQueueProducer 16 | 17 | Gvk schema.GroupVersionKind 18 | } 19 | 20 | func (g *GenericHandler) OnAdd(obj interface{}) { 21 | logger := g.Logger.With(logz.Operation("added")) 22 | g.add(logger, obj.(meta_v1.Object)) 23 | } 24 | 25 | func (g *GenericHandler) OnUpdate(oldObj, newObj interface{}) { 26 | logger := g.Logger.With(logz.Operation("updated")) 27 | g.add(logger, newObj.(meta_v1.Object)) 28 | } 29 | 30 | func (g *GenericHandler) OnDelete(obj interface{}) { 31 | metaObj, ok := obj.(meta_v1.Object) 32 | logger := g.Logger.With(logz.Operation("deleted")) 33 | if !ok { 34 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 35 | if !ok { 36 | logger.Sugar().Errorf("Delete event with unrecognized object type: %T", obj) 37 | return 38 | } 39 | metaObj, ok = tombstone.Obj.(meta_v1.Object) 40 | if !ok { 41 | logger.Sugar().Errorf("Delete tombstone with unrecognized object type: %T", tombstone.Obj) 42 | return 43 | } 44 | } 45 | g.add(logger, metaObj) 46 | } 47 | 48 | func (g *GenericHandler) add(logger *zap.Logger, obj meta_v1.Object) { 49 | g.loggerForObj(logger, obj).Info("Enqueuing object") 50 | g.WorkQueue.Add(ctrl.QueueKey{ 51 | Namespace: obj.GetNamespace(), 52 | Name: obj.GetName(), 53 | }) 54 | } 55 | 56 | func (g *GenericHandler) loggerForObj(logger *zap.Logger, obj meta_v1.Object) *zap.Logger { 57 | return logger.With(logz.Namespace(obj), 58 | logz.Object(obj), 59 | logz.ObjectGk(g.Gvk.GroupKind())) 60 | } 61 | -------------------------------------------------------------------------------- /process/work_queue.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/atlassian/ctrl" 8 | "k8s.io/apimachinery/pkg/runtime/schema" 9 | "k8s.io/client-go/util/workqueue" 10 | ) 11 | 12 | type gvkQueueKey struct { 13 | gvk schema.GroupVersionKind 14 | ctrl.QueueKey 15 | } 16 | 17 | func (g *gvkQueueKey) String() string { 18 | return fmt.Sprintf("%s, Ns=%s, N=%s", g.gvk, g.Namespace, g.Name) 19 | } 20 | 21 | // workQueue is a type safe wrapper around workqueue.RateLimitingInterface. 22 | type workQueue struct { 23 | // Objects that need to be synced. 24 | queue workqueue.RateLimitingInterface 25 | workDeduplicationPeriod time.Duration 26 | } 27 | 28 | func (q *workQueue) shutDown() { 29 | q.queue.ShutDown() 30 | } 31 | 32 | func (q *workQueue) get() (item gvkQueueKey, shutdown bool) { 33 | i, s := q.queue.Get() 34 | if s { 35 | return gvkQueueKey{}, true 36 | } 37 | return i.(gvkQueueKey), false 38 | } 39 | 40 | func (q *workQueue) done(item gvkQueueKey) { 41 | q.queue.Done(item) 42 | } 43 | 44 | func (q *workQueue) forget(item gvkQueueKey) { 45 | q.queue.Forget(item) 46 | } 47 | 48 | func (q *workQueue) numRequeues(item gvkQueueKey) int { 49 | return q.queue.NumRequeues(item) 50 | } 51 | 52 | func (q *workQueue) addRateLimited(item gvkQueueKey) { 53 | q.queue.AddRateLimited(item) 54 | } 55 | 56 | func (q *workQueue) newQueueForGvk(gvk schema.GroupVersionKind) *gvkQueue { 57 | return &gvkQueue{ 58 | queue: q.queue, 59 | gvk: gvk, 60 | workDeduplicationPeriod: q.workDeduplicationPeriod, 61 | } 62 | } 63 | 64 | type gvkQueue struct { 65 | queue workqueue.RateLimitingInterface 66 | gvk schema.GroupVersionKind 67 | workDeduplicationPeriod time.Duration 68 | } 69 | 70 | func (q *gvkQueue) Add(item ctrl.QueueKey) { 71 | q.queue.AddAfter(gvkQueueKey{ 72 | gvk: q.gvk, 73 | QueueKey: item, 74 | }, q.workDeduplicationPeriod) 75 | } 76 | -------------------------------------------------------------------------------- /apis/condition/v1/types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | ) 9 | 10 | type ConditionType string 11 | 12 | // These are some possible conditions of an object. 13 | const ( 14 | ConditionInProgress ConditionType = "InProgress" 15 | ConditionReady ConditionType = "Ready" 16 | ConditionError ConditionType = "Error" 17 | ) 18 | 19 | type ConditionStatus string 20 | 21 | // These are some possible condition statuses. "ConditionTrue" means a resource is in the condition. 22 | // "ConditionFalse" means a resource is not in the condition. "ConditionUnknown" means kubernetes 23 | // can't decide if a resource is in the condition or not. In the future, we could add other 24 | // intermediate conditions, e.g. ConditionDegraded. 25 | const ( 26 | ConditionTrue ConditionStatus = "True" 27 | ConditionFalse ConditionStatus = "False" 28 | ConditionUnknown ConditionStatus = "Unknown" 29 | ) 30 | 31 | // +k8s:deepcopy-gen=true 32 | // Condition describes the state of an object at a certain point. 33 | type Condition struct { 34 | // Type of State condition. 35 | Type ConditionType `json:"type"` 36 | // Status of the condition. 37 | Status ConditionStatus `json:"status"` 38 | // Last time the condition transitioned from one status to another. 39 | LastTransitionTime meta_v1.Time `json:"lastTransitionTime,omitempty"` 40 | // The reason for the condition's last transition. 41 | Reason string `json:"reason,omitempty"` 42 | // A human readable message indicating details about the transition. 43 | Message string `json:"message,omitempty"` 44 | } 45 | 46 | func (c *Condition) String() string { 47 | var buf bytes.Buffer 48 | buf.WriteString(string(c.Type)) // nolint: gosec 49 | buf.WriteByte(' ') // nolint: gosec 50 | buf.WriteString(string(c.Status)) // nolint: gosec 51 | if c.Reason != "" { 52 | fmt.Fprintf(&buf, " %q", c.Reason) // nolint: errcheck 53 | } 54 | if c.Message != "" { 55 | fmt.Fprintf(&buf, " %q", c.Message) // nolint: errcheck 56 | } 57 | return buf.String() 58 | } 59 | -------------------------------------------------------------------------------- /logz/logz.go: -------------------------------------------------------------------------------- 1 | package logz 2 | 3 | import ( 4 | "github.com/atlassian/ctrl" 5 | "go.uber.org/zap" 6 | "go.uber.org/zap/zapcore" 7 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/runtime/schema" 9 | ) 10 | 11 | // Object returns a zap field used to record ObjectName. 12 | func Object(obj meta_v1.Object) zapcore.Field { 13 | return ObjectName(obj.GetName()) 14 | } 15 | 16 | // ObjectName is a zap field to identify logs with the object name of a specific 17 | // object being processed in the ResourceEventHandler or in the Controller. 18 | func ObjectName(name string) zapcore.Field { 19 | return zap.String("obj_name", name) 20 | } 21 | 22 | // ObjectGk is a zap field to identify logs with the object gk of a specific 23 | // object being processed in the ResourceEventHandler or in the Controller. 24 | func ObjectGk(gk schema.GroupKind) zapcore.Field { 25 | return zap.Stringer("obj_gk", &gk) 26 | } 27 | 28 | // DelegateName is a zap field to identify logs of an object name that was processed in 29 | // the ResourceEventHandler, that had a lookup or owner that was queued instead. 30 | func DelegateName(name string) zapcore.Field { 31 | return zap.String("delegate_name", name) 32 | } 33 | 34 | // DelegateGk is a zap field to identify logs of an object gk that was processed in 35 | // the ResourceEventHandler, that had a lookup or owner that was queued instead. 36 | func DelegateGk(gk schema.GroupKind) zapcore.Field { 37 | return zap.Stringer("delegate_gk", &gk) 38 | } 39 | 40 | // Operation is a zap field used in ResourceEventHandler to identify the operation 41 | // that the logs are being produced from. 42 | func Operation(operation ctrl.Operation) zapcore.Field { 43 | return zap.Stringer("operation", operation) 44 | } 45 | 46 | func Namespace(obj meta_v1.Object) zapcore.Field { 47 | return NamespaceName(obj.GetNamespace()) 48 | } 49 | 50 | func NamespaceName(namespace string) zapcore.Field { 51 | if namespace == "" { 52 | return zap.Skip() 53 | } 54 | return zap.String("namespace", namespace) 55 | } 56 | 57 | func Iteration(iteration uint32) zapcore.Field { 58 | return zap.Uint32("iter", iteration) 59 | } 60 | -------------------------------------------------------------------------------- /flagutil/flags_test.go: -------------------------------------------------------------------------------- 1 | package flagutil 2 | 3 | import ( 4 | "flag" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestValidateFlags(t *testing.T) { 11 | t.Parallel() 12 | 13 | type config struct { 14 | f bool 15 | s string 16 | } 17 | 18 | cases := []struct { 19 | name string 20 | args []string 21 | valid bool 22 | error string 23 | }{ 24 | { 25 | name: "inline boolean", 26 | args: []string{"-f=true", "-s", "string"}, 27 | valid: true, 28 | }, 29 | { 30 | name: "boolean with no value", 31 | args: []string{"-f", "-s", "string"}, 32 | valid: true, 33 | }, 34 | { 35 | name: "last flag is boolean", 36 | args: []string{"-s", "string", "-f"}, 37 | valid: true, 38 | }, 39 | { 40 | name: "double dashes", 41 | args: []string{"--f=true", "--s", "string"}, 42 | valid: true, 43 | }, 44 | { 45 | name: "invalid boolean flag (true)", 46 | args: []string{"-f", "true", "-s", "string"}, 47 | valid: false, 48 | error: `invalid value following flag -f: "true"; boolean flags must be passed as -flag=x`, 49 | }, 50 | { 51 | name: "invalid boolean flag (false)", 52 | args: []string{"-f", "false", "-s", "string"}, 53 | valid: false, 54 | error: `invalid value following flag -f: "false"; boolean flags must be passed as -flag=x`, 55 | }, 56 | { 57 | name: "invalid boolean flag (arbitrary string)", 58 | args: []string{"-f", "bla", "-s", "string"}, 59 | valid: false, 60 | error: `invalid value following flag -f: "bla"; boolean flags must be passed as -flag=x`, 61 | }, 62 | { 63 | name: "invalid 'crap' flag", 64 | args: []string{"crap", "-f", "-s", "string"}, 65 | valid: false, 66 | error: "invalid flag: crap", 67 | }, 68 | } 69 | 70 | for _, c := range cases { 71 | c := c 72 | t.Run(c.name, func(t *testing.T) { 73 | t.Parallel() 74 | 75 | cfg := config{} 76 | fs := flag.NewFlagSet("test", flag.PanicOnError) 77 | fs.BoolVar(&cfg.f, "f", false, "boolean flag") 78 | fs.StringVar(&cfg.s, "s", "", "string flag") 79 | 80 | err := ValidateFlags(fs, c.args) 81 | if c.valid { 82 | require.NoError(t, err, "unexpected error: %#v", err) 83 | } else { 84 | require.Error(t, err, "expected validation error to be returned") 85 | require.Equal(t, c.error, err.Error(), "expected error: %q, but got: %q", c.error, err.Error()) 86 | } 87 | }) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /app/aux_server.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | "net/http/pprof" 8 | "time" 9 | 10 | "github.com/atlassian/ctrl/process" 11 | "github.com/go-chi/chi" 12 | "github.com/go-chi/chi/middleware" 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | const ( 19 | defaultMaxRequestDuration = 15 * time.Second 20 | shutdownTimeout = defaultMaxRequestDuration 21 | readTimeout = 1 * time.Second 22 | writeTimeout = defaultMaxRequestDuration 23 | idleTimeout = 1 * time.Minute 24 | ) 25 | 26 | type AuxServer struct { 27 | Logger *zap.Logger 28 | // Name is the name of the application. 29 | Name string 30 | Addr string // TCP address to listen on, ":http" if empty 31 | Gatherer prometheus.Gatherer 32 | IsReady func() bool 33 | Debug bool 34 | } 35 | 36 | func (a *AuxServer) Run(ctx context.Context) error { 37 | if a.Addr == "" { 38 | <-ctx.Done() 39 | return nil 40 | } 41 | srv := &http.Server{ 42 | Addr: a.Addr, 43 | Handler: a.constructHandler(), 44 | WriteTimeout: writeTimeout, 45 | ReadTimeout: readTimeout, 46 | IdleTimeout: idleTimeout, 47 | } 48 | return process.StartStopServer(ctx, srv, shutdownTimeout) 49 | } 50 | 51 | func (a *AuxServer) constructHandler() *chi.Mux { 52 | router := chi.NewRouter() 53 | router.Use(middleware.Timeout(defaultMaxRequestDuration), a.setServerHeader) 54 | router.NotFound(pageNotFound) 55 | 56 | router.Method(http.MethodGet, "/metrics", promhttp.HandlerFor(a.Gatherer, promhttp.HandlerOpts{})) 57 | router.Get("/healthz/ping", func(_ http.ResponseWriter, _ *http.Request) {}) 58 | router.Get("/healthz/ready", func(w http.ResponseWriter, _ *http.Request) { 59 | if !a.IsReady() { 60 | w.WriteHeader(http.StatusServiceUnavailable) 61 | io.WriteString(w, "Not ready") // nolint: errcheck, gosec 62 | return 63 | } 64 | w.WriteHeader(http.StatusOK) 65 | }) 66 | if a.Debug { 67 | // Enable debug endpoints 68 | router.HandleFunc("/debug/pprof/", pprof.Index) 69 | router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 70 | router.HandleFunc("/debug/pprof/profile", pprof.Profile) 71 | router.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 72 | router.HandleFunc("/debug/pprof/trace", pprof.Trace) 73 | } 74 | 75 | return router 76 | } 77 | 78 | func (a *AuxServer) setServerHeader(next http.Handler) http.Handler { 79 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 80 | w.Header().Set("Server", a.Name) 81 | next.ServeHTTP(w, r) 82 | }) 83 | } 84 | 85 | func pageNotFound(w http.ResponseWriter, _ *http.Request) { 86 | w.WriteHeader(http.StatusNotFound) 87 | } 88 | -------------------------------------------------------------------------------- /handlers/lookup_handler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/atlassian/ctrl" 5 | "github.com/atlassian/ctrl/logz" 6 | "go.uber.org/zap" 7 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | "k8s.io/apimachinery/pkg/runtime/schema" 10 | "k8s.io/client-go/tools/cache" 11 | ) 12 | 13 | // LookupHandler is a handler for controlled objects that can be mapped to some controller object 14 | // through the use of a Lookup function. 15 | // This handler assumes that the Logger already has the ctrl_gk field set. 16 | type LookupHandler struct { 17 | Logger *zap.Logger 18 | WorkQueue ctrl.WorkQueueProducer 19 | Gvk schema.GroupVersionKind 20 | 21 | Lookup func(runtime.Object) ([]runtime.Object, error) 22 | } 23 | 24 | func (e *LookupHandler) enqueueMapped(logger *zap.Logger, obj meta_v1.Object) { 25 | logger = e.loggerForObj(logger, obj) 26 | objs, err := e.Lookup(obj.(runtime.Object)) 27 | if err != nil { 28 | logger.Error("Failed to lookup objects", zap.Error(err)) 29 | return 30 | } 31 | if len(objs) == 0 { 32 | logger.Debug("Lookup function returned zero results") 33 | } 34 | for _, o := range objs { 35 | metaobj := o.(meta_v1.Object) 36 | logger. 37 | With(logz.DelegateName(obj.GetName())). 38 | With(logz.DelegateGk(e.Gvk.GroupKind())). 39 | Info("Enqueuing looked up object") 40 | e.WorkQueue.Add(ctrl.QueueKey{ 41 | Namespace: metaobj.GetNamespace(), 42 | Name: metaobj.GetName(), 43 | }) 44 | } 45 | } 46 | 47 | func (e *LookupHandler) OnAdd(obj interface{}) { 48 | logger := e.Logger.With(logz.Operation(ctrl.AddedOperation)) 49 | e.enqueueMapped(logger, obj.(meta_v1.Object)) 50 | } 51 | 52 | func (e *LookupHandler) OnUpdate(oldObj, newObj interface{}) { 53 | logger := e.Logger.With(logz.Operation(ctrl.UpdatedOperation)) 54 | e.enqueueMapped(logger, newObj.(meta_v1.Object)) 55 | } 56 | 57 | func (e *LookupHandler) OnDelete(obj interface{}) { 58 | metaObj, ok := obj.(meta_v1.Object) 59 | logger := e.Logger.With(logz.Operation(ctrl.DeletedOperation)) 60 | if !ok { 61 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 62 | if !ok { 63 | logger.Sugar().Errorf("Delete event with unrecognized object type: %T", obj) 64 | return 65 | } 66 | metaObj, ok = tombstone.Obj.(meta_v1.Object) 67 | if !ok { 68 | logger.Sugar().Errorf("Delete tombstone with unrecognized object type: %T", tombstone.Obj) 69 | return 70 | } 71 | } 72 | e.enqueueMapped(logger, metaObj) 73 | } 74 | 75 | // loggerForObj returns a logger with fields for a controlled object. 76 | func (e *LookupHandler) loggerForObj(logger *zap.Logger, obj meta_v1.Object) *zap.Logger { 77 | return logger.With(logz.Namespace(obj), 78 | logz.Object(obj), 79 | logz.ObjectGk(e.Gvk.GroupKind())) 80 | } 81 | -------------------------------------------------------------------------------- /options/client.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "github.com/atlassian/ctrl" 5 | "github.com/pkg/errors" 6 | "k8s.io/client-go/rest" 7 | "k8s.io/client-go/tools/clientcmd" 8 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 9 | "k8s.io/client-go/util/flowcontrol" 10 | ) 11 | 12 | const ( 13 | DefaultAPIQPS = 5 14 | APIQPSBurstFactor = 1.5 15 | ) 16 | 17 | type RestClientOptions struct { 18 | APIQPS float64 19 | ClientConfigFileFrom string 20 | ClientConfigFileName string 21 | ClientContext string 22 | } 23 | 24 | func (o *RestClientOptions) DefaultAndValidate() []error { 25 | var allErrors []error 26 | if o.APIQPS == 0 { 27 | o.APIQPS = DefaultAPIQPS 28 | } 29 | if o.APIQPS < 0 { 30 | allErrors = append(allErrors, errors.Errorf("value for API QPS must be non-negative. Given: %f", o.APIQPS)) 31 | } 32 | return allErrors 33 | } 34 | 35 | func BindRestClientFlags(o *RestClientOptions, fs ctrl.FlagSet) { 36 | fs.Float64Var(&o.APIQPS, "api-qps", DefaultAPIQPS, "Maximum queries per second when talking to Kubernetes API") 37 | fs.StringVar(&o.ClientConfigFileFrom, "client-config-from", "in-cluster", 38 | "Source of REST client configuration. 'in-cluster' (default) and 'file' are valid options.") 39 | fs.StringVar(&o.ClientConfigFileName, "client-config-file-name", "", 40 | "Load REST client configuration from the specified Kubernetes config file. This is only applicable if --client-config-from=file is set.") 41 | fs.StringVar(&o.ClientContext, "client-config-context", "", 42 | "Context to use for REST client configuration. This is only applicable if --client-config-from=file is set.") 43 | } 44 | 45 | func LoadRestClientConfig(userAgent string, options RestClientOptions) (*rest.Config, error) { 46 | var config *rest.Config 47 | var err error 48 | 49 | switch options.ClientConfigFileFrom { 50 | case "in-cluster": 51 | config, err = rest.InClusterConfig() 52 | case "file": 53 | var configAPI *clientcmdapi.Config 54 | configAPI, err = clientcmd.LoadFromFile(options.ClientConfigFileName) 55 | if err != nil { 56 | return nil, errors.Wrapf(err, "failed to load REST client configuration from file %q", options.ClientConfigFileName) 57 | } 58 | config, err = clientcmd.NewDefaultClientConfig(*configAPI, &clientcmd.ConfigOverrides{ 59 | CurrentContext: options.ClientContext, 60 | }).ClientConfig() 61 | default: 62 | err = errors.New("invalid value for 'client config from' parameter") 63 | } 64 | if err != nil { 65 | return nil, errors.Wrapf(err, "failed to load REST client configuration from %q", options.ClientConfigFileFrom) 66 | } 67 | config.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(options.APIQPS), int(options.APIQPS*APIQPSBurstFactor)) 68 | config.UserAgent = userAgent 69 | return config, nil 70 | } 71 | -------------------------------------------------------------------------------- /flagutil/flags.go: -------------------------------------------------------------------------------- 1 | package flagutil 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | ) 8 | 9 | func ValidateFlags(flagset *flag.FlagSet, args []string) error { 10 | validator := flagValidator{ 11 | args: args, 12 | flagset: flagset, 13 | } 14 | for { 15 | seen, err := validator.validateNextFlag() 16 | if seen { 17 | continue 18 | } 19 | if err == nil { 20 | break 21 | } 22 | return err 23 | } 24 | return nil 25 | } 26 | 27 | type flagValidator struct { 28 | args []string 29 | flagset *flag.FlagSet 30 | } 31 | 32 | // optional interface to indicate boolean flags that can be 33 | // supplied without "=value" text 34 | type boolFlag interface { 35 | IsBoolFlag() bool 36 | } 37 | 38 | // This method is based on the parseOne() method from https://golang.org/src/flag/flag.go, 39 | // but was rewritten for extra validation rather than parsing flag values. 40 | // nolint: gocyclo 41 | func (v *flagValidator) validateNextFlag() (bool, error) { 42 | if len(v.args) == 0 { 43 | return false, nil 44 | } 45 | s := v.args[0] 46 | if len(s) < 2 || s[0] != '-' { 47 | return false, fmt.Errorf("invalid flag: %s", s) 48 | } 49 | numMinuses := 1 50 | if s[1] == '-' { 51 | numMinuses++ 52 | if len(s) == 2 { // "--" terminates the flags 53 | v.args = v.args[1:] 54 | return false, errors.New("'--' flag with no name is not allowed") 55 | } 56 | } 57 | name := s[numMinuses:] 58 | if len(name) == 0 || name[0] == '-' || name[0] == '=' { 59 | return false, fmt.Errorf("bad flag syntax: %s", s) 60 | } 61 | 62 | // it's a flag. does it have an argument? 63 | v.args = v.args[1:] 64 | hasValue := false 65 | value := "" 66 | for i := 1; i < len(name); i++ { // equals cannot be first 67 | if name[i] == '=' { 68 | value = name[i+1:] 69 | hasValue = true 70 | name = name[0:i] 71 | break 72 | } 73 | } 74 | 75 | flag := v.flagset.Lookup(name) 76 | if flag == nil { 77 | return false, fmt.Errorf("undefined flag: -%s", name) 78 | } 79 | 80 | if fv, ok := flag.Value.(boolFlag); ok && fv.IsBoolFlag() { // special case: doesn't need an arg 81 | if len(v.args) > 0 { 82 | nextArg := v.args[0] 83 | if nextArg[0] != '-' { 84 | return false, fmt.Errorf("invalid value following flag -%s: %q; boolean flags must be passed as -flag=x", name, nextArg) 85 | } 86 | } 87 | } else { 88 | // It must have a value, which might be the next argument. 89 | if !hasValue && len(v.args) > 0 { 90 | // value is the next arg 91 | hasValue = true 92 | value, v.args = v.args[0], v.args[1:] 93 | } 94 | if !hasValue { 95 | return false, fmt.Errorf("flag needs an argument: -%s", name) 96 | } 97 | } 98 | _ = value 99 | return true, nil 100 | } 101 | -------------------------------------------------------------------------------- /process/generic_worker.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "strconv" 5 | "sync/atomic" 6 | "time" 7 | 8 | "github.com/atlassian/ctrl" 9 | "github.com/atlassian/ctrl/logz" 10 | "github.com/pkg/errors" 11 | "go.uber.org/zap" 12 | api_errors "k8s.io/apimachinery/pkg/api/errors" 13 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/apimachinery/pkg/runtime" 15 | "k8s.io/apimachinery/pkg/runtime/schema" 16 | "k8s.io/client-go/tools/cache" 17 | ) 18 | 19 | const ( 20 | // maxRetries is the number of times an object will be retried before it is dropped out of the queue. 21 | // With the current rate-limiter in use (5ms*2^(maxRetries-1)) the following numbers represent the times 22 | // an object is going to be requeued: 23 | // 24 | // 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s 25 | maxRetries = 15 26 | ) 27 | 28 | func (g *Generic) worker() { 29 | for g.processNextWorkItem() { 30 | } 31 | } 32 | 33 | func (g *Generic) processNextWorkItem() bool { 34 | key, quit := g.queue.get() 35 | if quit { 36 | return false 37 | } 38 | defer g.queue.done(key) 39 | 40 | holder := g.Controllers[key.gvk] 41 | logger := g.logger.With(logz.NamespaceName(key.Namespace), 42 | logz.ObjectName(key.Name), 43 | logz.ObjectGk(key.gvk.GroupKind()), 44 | logz.Iteration(atomic.AddUint32(&g.iter, 1))) 45 | 46 | external, retriable, err := g.processKey(logger, holder, key) 47 | g.handleErr(logger, holder, external, retriable, err, key) 48 | 49 | return true 50 | } 51 | 52 | func (g *Generic) handleErr(logger *zap.Logger, holder Holder, external bool, retriable bool, err error, key gvkQueueKey) { 53 | groupKind := key.gvk.GroupKind() 54 | 55 | if err == nil { 56 | g.queue.forget(key) 57 | return 58 | } 59 | 60 | if retriable && g.queue.numRequeues(key) < maxRetries { 61 | logger.Info("Error syncing object, will retry", zap.Error(err)) 62 | g.queue.addRateLimited(key) 63 | holder.objectProcessErrors. 64 | WithLabelValues(holder.AppName, key.Namespace, key.Name, groupKind.String(), strconv.FormatBool(external), strconv.FormatBool(true)). 65 | Inc() 66 | return 67 | } 68 | 69 | holder.objectProcessErrors. 70 | WithLabelValues(holder.AppName, key.Namespace, key.Name, groupKind.String(), strconv.FormatBool(external), strconv.FormatBool(false)). 71 | Inc() 72 | 73 | if external { 74 | logger.Info("Dropping object out of the queue due to external error", zap.Error(err)) 75 | } else { 76 | logger.Error("Dropping object out of the queue due to internal error", zap.Error(err)) 77 | } 78 | g.queue.forget(key) 79 | } 80 | 81 | func (g *Generic) processKey(logger *zap.Logger, holder Holder, key gvkQueueKey) (bool /* external */, bool /*retriable*/, error) { 82 | groupKind := key.gvk.GroupKind() 83 | 84 | cntrlr := holder.Cntrlr 85 | informer := g.Informers[key.gvk] 86 | obj, exists, err := getFromIndexer(informer.GetIndexer(), key.gvk, key.Namespace, key.Name) 87 | if err != nil { 88 | return false, false, errors.Wrapf(err, "failed to get object by key %s", key.String()) 89 | } 90 | if !exists { 91 | logger.Debug("Object not in cache. Was deleted?") 92 | return false, false, nil 93 | } 94 | startTime := time.Now() 95 | logger.Info("Started syncing") 96 | 97 | msg := "" 98 | defer func() { 99 | totalTime := time.Since(startTime) 100 | holder.objectProcessTime.WithLabelValues(holder.AppName, key.Namespace, key.Name, groupKind.String()).Observe(totalTime.Seconds()) 101 | logger.Sugar().Infof("Synced in %v%s", totalTime, msg) 102 | }() 103 | 104 | external, retriable, err := cntrlr.Process(&ctrl.ProcessContext{ 105 | Logger: logger, 106 | Object: obj, 107 | }) 108 | if err != nil && api_errors.IsConflict(errors.Cause(err)) { 109 | msg = " (conflict)" 110 | err = nil 111 | } 112 | return external, retriable, err 113 | } 114 | 115 | func getFromIndexer(indexer cache.Indexer, gvk schema.GroupVersionKind, namespace, name string) (runtime.Object, bool /*exists */, error) { 116 | obj, exists, err := indexer.GetByKey(ByNamespaceAndNameIndexKey(namespace, name)) 117 | if err != nil || !exists { 118 | return nil, exists, err 119 | } 120 | ro := obj.(runtime.Object).DeepCopyObject() 121 | ro.GetObjectKind().SetGroupVersionKind(gvk) // Objects from type-specific informers don't have GVK set 122 | return ro, true, nil 123 | } 124 | 125 | func ByNamespaceAndNameIndexKey(namespace, name string) string { 126 | if namespace == meta_v1.NamespaceNone { 127 | return name 128 | } 129 | return namespace + "/" + name 130 | } 131 | -------------------------------------------------------------------------------- /options/leader_election.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "time" 7 | 8 | "github.com/atlassian/ctrl" 9 | "github.com/atlassian/ctrl/logz" 10 | "go.uber.org/zap" 11 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | core_v1client "k8s.io/client-go/kubernetes/typed/core/v1" 13 | "k8s.io/client-go/tools/leaderelection" 14 | "k8s.io/client-go/tools/leaderelection/resourcelock" 15 | "k8s.io/client-go/tools/record" 16 | ) 17 | 18 | const ( 19 | defaultLeaseDuration = 15 * time.Second 20 | defaultRenewDeadline = 10 * time.Second 21 | defaultRetryPeriod = 2 * time.Second 22 | ) 23 | 24 | // See k8s.io/apiserver/pkg/apis/config/types.go LeaderElectionConfiguration 25 | // for leader election configuration description. 26 | type LeaderElectionOptions struct { 27 | LeaderElect bool 28 | LeaseDuration time.Duration 29 | RenewDeadline time.Duration 30 | RetryPeriod time.Duration 31 | ConfigMapNamespace string 32 | ConfigMapName string 33 | } 34 | 35 | // DoLeaderElection starts leader election and blocks until it acquires the lease. 36 | // Returned context is cancelled once the lease is lost or ctx signals done. 37 | func DoLeaderElection(ctx context.Context, logger *zap.Logger, component string, config LeaderElectionOptions, configMapsGetter core_v1client.ConfigMapsGetter, recorder record.EventRecorder) (context.Context, error) { 38 | id, err := os.Hostname() 39 | if err != nil { 40 | return nil, err 41 | } 42 | ctxRet, cancel := context.WithCancel(ctx) 43 | startedLeading := make(chan struct{}) 44 | le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ 45 | Lock: &resourcelock.ConfigMapLock{ 46 | ConfigMapMeta: meta_v1.ObjectMeta{ 47 | Namespace: config.ConfigMapNamespace, 48 | Name: config.ConfigMapName, 49 | }, 50 | Client: configMapsGetter, 51 | LockConfig: resourcelock.ResourceLockConfig{ 52 | Identity: id + "-" + component, 53 | EventRecorder: recorder, 54 | }, 55 | }, 56 | LeaseDuration: config.LeaseDuration, 57 | RenewDeadline: config.RenewDeadline, 58 | RetryPeriod: config.RetryPeriod, 59 | Callbacks: leaderelection.LeaderCallbacks{ 60 | OnStartedLeading: func(ctx context.Context) { 61 | logger.Info("Started leading") 62 | close(startedLeading) 63 | }, 64 | OnStoppedLeading: func() { 65 | logger.Info("Leader status lost") 66 | cancel() 67 | }, 68 | }, 69 | }) 70 | if err != nil { 71 | cancel() 72 | return nil, err 73 | } 74 | go func() { 75 | // note: because le.Run() also adds a logging panic handler panics with be logged 3 times 76 | defer logz.LogStructuredPanic() 77 | le.Run(ctx) 78 | }() 79 | select { 80 | case <-ctx.Done(): 81 | return nil, ctx.Err() 82 | case <-startedLeading: 83 | return ctxRet, nil 84 | } 85 | } 86 | 87 | func BindLeaderElectionFlags(component string, o *LeaderElectionOptions, fs ctrl.FlagSet) { 88 | // This flag is off by default only because leader election package says it is ALPHA API. 89 | fs.BoolVar(&o.LeaderElect, "leader-elect", false, ""+ 90 | "Start a leader election client and gain leadership before "+ 91 | "executing the main loop. Enable this when running replicated "+ 92 | "components for high availability") 93 | fs.DurationVar(&o.LeaseDuration, "leader-elect-lease-duration", defaultLeaseDuration, ""+ 94 | "The duration that non-leader candidates will wait after observing a leadership "+ 95 | "renewal until attempting to acquire leadership of a led but unrenewed leader "+ 96 | "slot. This is effectively the maximum duration that a leader can be stopped "+ 97 | "before it is replaced by another candidate. This is only applicable if leader "+ 98 | "election is enabled") 99 | fs.DurationVar(&o.RenewDeadline, "leader-elect-renew-deadline", defaultRenewDeadline, ""+ 100 | "The interval between attempts by the acting master to renew a leadership slot "+ 101 | "before it stops leading. This must be less than or equal to the lease duration. "+ 102 | "This is only applicable if leader election is enabled") 103 | fs.DurationVar(&o.RetryPeriod, "leader-elect-retry-period", defaultRetryPeriod, ""+ 104 | "The duration the clients should wait between attempting acquisition and renewal "+ 105 | "of a leadership. This is only applicable if leader election is enabled") 106 | fs.StringVar(&o.ConfigMapNamespace, "leader-elect-configmap-namespace", meta_v1.NamespaceDefault, 107 | "Namespace to use for leader election ConfigMap. This is only applicable if leader election is enabled") 108 | fs.StringVar(&o.ConfigMapName, "leader-elect-configmap-name", component+"-leader-elect", 109 | "ConfigMap name to use for leader election. This is only applicable if leader election is enabled") 110 | } 111 | -------------------------------------------------------------------------------- /handlers/controlled_resource_handler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/atlassian/ctrl" 5 | "github.com/atlassian/ctrl/logz" 6 | "go.uber.org/zap" 7 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | "k8s.io/apimachinery/pkg/runtime/schema" 10 | "k8s.io/client-go/tools/cache" 11 | ) 12 | 13 | // ControllerIndex is an index from controlled to controller objects. 14 | type ControllerIndex interface { 15 | // ControllerByObject returns controller objects that own or want to own an object with a particular Group, Kind, 16 | // namespace and name. "want to own" means that the object might not exist yet but the controller 17 | // object would want it to. 18 | ControllerByObject(gk schema.GroupKind, namespace, name string) ([]runtime.Object, error) 19 | } 20 | 21 | // ControlledResourceHandler is a handler for objects the are controlled/owned/produced by some controller object. 22 | // The controller object is identified by a controller owner reference on the controlled objects. 23 | // This handler assumes that: 24 | // - Logger already has the cntrl_gk field set. 25 | // - controlled and controller objects exist in the same namespace and never across namespaces. 26 | type ControlledResourceHandler struct { 27 | Logger *zap.Logger 28 | WorkQueue ctrl.WorkQueueProducer 29 | ControllerIndex ControllerIndex 30 | ControllerGvk schema.GroupVersionKind 31 | Gvk schema.GroupVersionKind 32 | } 33 | 34 | func (g *ControlledResourceHandler) enqueueMapped(logger *zap.Logger, metaObj meta_v1.Object) { 35 | name, namespace := g.getControllerNameAndNamespace(metaObj) 36 | logger = g.loggerForObj(logger, metaObj) 37 | 38 | if name == "" { 39 | if g.ControllerIndex != nil { 40 | controllers, err := g.ControllerIndex.ControllerByObject( 41 | metaObj.(runtime.Object).GetObjectKind().GroupVersionKind().GroupKind(), namespace, metaObj.GetName()) 42 | if err != nil { 43 | logger.Error("Failed to get controllers for object", zap.Error(err)) 44 | return 45 | } 46 | for _, controller := range controllers { 47 | controllerMeta := controller.(meta_v1.Object) 48 | g.rebuildControllerByName(logger, controllerMeta.GetNamespace(), controllerMeta.GetName()) 49 | } 50 | } 51 | } else { 52 | g.rebuildControllerByName(logger, namespace, name) 53 | } 54 | } 55 | 56 | func (g *ControlledResourceHandler) OnAdd(obj interface{}) { 57 | metaObj := obj.(meta_v1.Object) 58 | logger := g.Logger.With(logz.Operation(ctrl.AddedOperation)) 59 | g.enqueueMapped(logger, metaObj) 60 | } 61 | 62 | func (g *ControlledResourceHandler) OnUpdate(oldObj, newObj interface{}) { 63 | oldMeta := oldObj.(meta_v1.Object) 64 | newMeta := newObj.(meta_v1.Object) 65 | logger := g.Logger.With(logz.Operation(ctrl.UpdatedOperation)) 66 | 67 | oldName, _ := g.getControllerNameAndNamespace(oldMeta) 68 | newName, _ := g.getControllerNameAndNamespace(newMeta) 69 | 70 | if oldName != newName { 71 | g.enqueueMapped(logger, oldMeta) 72 | } 73 | 74 | g.enqueueMapped(logger, newMeta) 75 | } 76 | 77 | func (g *ControlledResourceHandler) OnDelete(obj interface{}) { 78 | metaObj, ok := obj.(meta_v1.Object) 79 | logger := g.Logger.With(logz.Operation(ctrl.DeletedOperation)) 80 | if !ok { 81 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 82 | if !ok { 83 | logger.Sugar().Errorf("Delete event with unrecognized object type: %T", obj) 84 | return 85 | } 86 | metaObj, ok = tombstone.Obj.(meta_v1.Object) 87 | if !ok { 88 | logger.Sugar().Errorf("Delete tombstone with unrecognized object type: %T", tombstone.Obj) 89 | return 90 | } 91 | } 92 | g.enqueueMapped(logger, metaObj) 93 | } 94 | 95 | // This method may be called with an empty controllerName. 96 | func (g *ControlledResourceHandler) rebuildControllerByName(logger *zap.Logger, namespace, controllerName string) { 97 | if controllerName == "" { 98 | logger.Debug("Object has no controller, so nothing was enqueued") 99 | return 100 | } 101 | logger. 102 | With(logz.DelegateName(controllerName)). 103 | With(logz.DelegateGk(g.ControllerGvk.GroupKind())). 104 | Info("Enqueuing controller") 105 | g.WorkQueue.Add(ctrl.QueueKey{ 106 | Namespace: namespace, 107 | Name: controllerName, 108 | }) 109 | } 110 | 111 | // getControllerNameAndNamespace returns name and namespace of the object's controller. 112 | // Returned name may be empty if the object does not have a controller owner reference. 113 | func (g *ControlledResourceHandler) getControllerNameAndNamespace(obj meta_v1.Object) (string, string) { 114 | var name string 115 | ref := meta_v1.GetControllerOf(obj) 116 | if ref != nil && ref.APIVersion == g.ControllerGvk.GroupVersion().String() && ref.Kind == g.ControllerGvk.Kind { 117 | name = ref.Name 118 | } 119 | return name, obj.GetNamespace() 120 | } 121 | 122 | func (g *ControlledResourceHandler) loggerForObj(logger *zap.Logger, obj meta_v1.Object) *zap.Logger { 123 | return logger.With(logz.Namespace(obj), 124 | logz.Object(obj), 125 | logz.ObjectGk(g.Gvk.GroupKind())) 126 | } 127 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package ctrl 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "go.uber.org/zap" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | "k8s.io/client-go/kubernetes" 14 | "k8s.io/client-go/rest" 15 | "k8s.io/client-go/tools/cache" 16 | ) 17 | 18 | type FlagSet interface { 19 | DurationVar(p *time.Duration, name string, value time.Duration, usage string) 20 | IntVar(p *int, name string, value int, usage string) 21 | Float64Var(p *float64, name string, value float64, usage string) 22 | StringVar(p *string, name string, value string, usage string) 23 | BoolVar(p *bool, name string, value bool, usage string) 24 | UintVar(p *uint, name string, value uint, usage string) 25 | Int64Var(p *int64, name string, value int64, usage string) 26 | Uint64Var(p *uint64, name string, value uint64, usage string) 27 | } 28 | 29 | type Descriptor struct { 30 | // Group Version Kind of objects a controller can process. 31 | Gvk schema.GroupVersionKind 32 | } 33 | 34 | type Server interface { 35 | Run(context.Context) error 36 | } 37 | 38 | type Constructed struct { 39 | // Interface holds an optional controller interface. 40 | Interface Interface 41 | // Server holds an optional server interface. 42 | Server Server 43 | } 44 | 45 | type Constructor interface { 46 | AddFlags(FlagSet) 47 | // New constructs a new controller and/or server. 48 | // If it constructs a controller, it must register an informer for the GVK controller 49 | // handles via Context.RegisterInformer(). 50 | New(*Config, *Context) (*Constructed, error) 51 | Describe() Descriptor 52 | } 53 | 54 | type Interface interface { 55 | Run(context.Context) 56 | 57 | // Process is implemented by the controller and returns: 58 | // - true for externalErr if the error is not an internal error 59 | // - true for retriableErr if the error is a retriable error (i.e. should be 60 | // added back to the work queue). These are retried for limited number of 61 | // attempts 62 | // - an error, if there is an error, or nil. If there is no error, the above 63 | // two bools (externalErr and retriableErr) are ignored. 64 | Process(*ProcessContext) (externalErr bool, retriableErr bool, err error) 65 | } 66 | 67 | type WorkQueueProducer interface { 68 | // Add adds an item to the workqueue. 69 | Add(QueueKey) 70 | } 71 | 72 | type ProcessContext struct { 73 | Logger *zap.Logger 74 | Object runtime.Object 75 | } 76 | 77 | type QueueKey struct { 78 | Namespace string 79 | Name string 80 | } 81 | 82 | type Config struct { 83 | AppName string 84 | Logger *zap.Logger 85 | Namespace string 86 | ResyncPeriod time.Duration 87 | Registry prometheus.Registerer 88 | 89 | RestConfig *rest.Config 90 | MainClient kubernetes.Interface 91 | } 92 | 93 | type Operation string 94 | 95 | const ( 96 | UpdatedOperation Operation = "updated" 97 | DeletedOperation Operation = "deleted" 98 | AddedOperation Operation = "added" 99 | ) 100 | 101 | func (o Operation) String() string { 102 | return string(o) 103 | } 104 | 105 | type Context struct { 106 | // ReadyForWork is a function that the controller must call from its Run() method once it is ready to 107 | // process work using it's Process() method. This should be used to delay processing while some initialization 108 | // is being performed. 109 | ReadyForWork func() 110 | // Middleware is the standard middleware that is supposed to be used to wrap the http handler of the server. 111 | Middleware func(http.Handler) http.Handler 112 | // Will contain all informers once Generic controller constructs all controllers. 113 | // This is a read only field, must not be modified. 114 | Informers map[schema.GroupVersionKind]cache.SharedIndexInformer 115 | // Will contain all controllers once Generic controller constructs them. 116 | // This is a read only field, must not be modified. 117 | Controllers map[schema.GroupVersionKind]Interface 118 | WorkQueue WorkQueueProducer 119 | } 120 | 121 | func (c *Context) RegisterInformer(gvk schema.GroupVersionKind, inf cache.SharedIndexInformer) error { 122 | if _, ok := c.Informers[gvk]; ok { 123 | return errors.New("informer with this GVK has been registered already") 124 | } 125 | if c.Informers == nil { 126 | c.Informers = make(map[schema.GroupVersionKind]cache.SharedIndexInformer) 127 | } 128 | c.Informers[gvk] = inf 129 | return nil 130 | } 131 | 132 | func (c *Context) MainInformer(config *Config, gvk schema.GroupVersionKind, f func(kubernetes.Interface, string, time.Duration, cache.Indexers) cache.SharedIndexInformer) (cache.SharedIndexInformer, error) { 133 | inf := c.Informers[gvk] 134 | if inf == nil { 135 | inf = f(config.MainClient, config.Namespace, config.ResyncPeriod, cache.Indexers{}) 136 | err := c.RegisterInformer(gvk, inf) 137 | if err != nil { 138 | return nil, err 139 | } 140 | } 141 | return inf, nil 142 | } 143 | 144 | func (c *Context) MainClusterInformer(config *Config, gvk schema.GroupVersionKind, f func(kubernetes.Interface, time.Duration, cache.Indexers) cache.SharedIndexInformer) (cache.SharedIndexInformer, error) { 145 | inf := c.Informers[gvk] 146 | if inf == nil { 147 | inf = f(config.MainClient, config.ResyncPeriod, cache.Indexers{}) 148 | err := c.RegisterInformer(gvk, inf) 149 | if err != nil { 150 | return nil, err 151 | } 152 | } 153 | return inf, nil 154 | } 155 | -------------------------------------------------------------------------------- /app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | 11 | "github.com/ash2k/stager" 12 | "github.com/atlassian/ctrl" 13 | "github.com/atlassian/ctrl/flagutil" 14 | "github.com/atlassian/ctrl/logz" 15 | "github.com/atlassian/ctrl/options" 16 | "github.com/atlassian/ctrl/process" 17 | "github.com/prometheus/client_golang/prometheus" 18 | "go.uber.org/zap" 19 | core_v1 "k8s.io/api/core/v1" 20 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | "k8s.io/apimachinery/pkg/util/errors" 23 | "k8s.io/client-go/kubernetes" 24 | core_v1client "k8s.io/client-go/kubernetes/typed/core/v1" 25 | "k8s.io/client-go/rest" 26 | "k8s.io/client-go/tools/record" 27 | "k8s.io/client-go/util/workqueue" 28 | ) 29 | 30 | const ( 31 | defaultAuxServerAddr = ":9090" 32 | ) 33 | 34 | type PrometheusRegistry interface { 35 | prometheus.Registerer 36 | prometheus.Gatherer 37 | } 38 | 39 | type App struct { 40 | Logger *zap.Logger 41 | 42 | options.GenericNamespacedControllerOptions 43 | options.LeaderElectionOptions 44 | options.RestClientOptions 45 | options.LoggerOptions 46 | 47 | MainClient kubernetes.Interface 48 | PrometheusRegistry PrometheusRegistry 49 | 50 | // Name is the name of the application. It must only contain alphanumeric 51 | // characters. 52 | Name string 53 | RestConfig *rest.Config 54 | Controllers []ctrl.Constructor 55 | AuxListenOn string 56 | Debug bool 57 | } 58 | 59 | func (a *App) Run(ctx context.Context) (retErr error) { 60 | defer func() { 61 | if err := a.Logger.Sync(); err != nil { 62 | fmt.Fprintf(os.Stderr, "Failed to flush (AKA sync) remaining logs: %v\n", err) // nolint: errcheck 63 | } 64 | }() 65 | 66 | // Controller 67 | config := &ctrl.Config{ 68 | AppName: a.Name, 69 | Namespace: a.Namespace, 70 | ResyncPeriod: a.ResyncPeriod, 71 | Registry: a.PrometheusRegistry, 72 | Logger: a.Logger, 73 | 74 | RestConfig: a.RestConfig, 75 | MainClient: a.MainClient, 76 | } 77 | generic, err := process.NewGeneric(config, 78 | workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "multiqueue"), 79 | a.Workers, a.Controllers...) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | // Auxiliary server 85 | auxSrv := AuxServer{ 86 | Logger: a.Logger, 87 | Addr: a.AuxListenOn, 88 | Gatherer: a.PrometheusRegistry, 89 | IsReady: generic.IsReady, 90 | Debug: a.Debug, 91 | } 92 | 93 | // Events 94 | eventsScheme := runtime.NewScheme() 95 | // we use ConfigMapLock which emits events for ConfigMap and hence we need (only) core_v1 types for it 96 | if err = core_v1.AddToScheme(eventsScheme); err != nil { 97 | return err 98 | } 99 | 100 | // Start events recorder 101 | eventBroadcaster := record.NewBroadcaster() 102 | loggingWatch := eventBroadcaster.StartLogging(a.Logger.Sugar().Infof) 103 | defer loggingWatch.Stop() 104 | recordingWatch := eventBroadcaster.StartRecordingToSink(&core_v1client.EventSinkImpl{Interface: a.MainClient.CoreV1().Events(meta_v1.NamespaceNone)}) 105 | defer recordingWatch.Stop() 106 | recorder := eventBroadcaster.NewRecorder(eventsScheme, core_v1.EventSource{Component: a.Name}) 107 | 108 | var auxErr error 109 | defer func() { 110 | if auxErr != nil && (retErr == context.DeadlineExceeded || retErr == context.Canceled) { 111 | retErr = auxErr 112 | } 113 | }() 114 | 115 | stgr := stager.New() 116 | defer stgr.Shutdown() 117 | stage := stgr.NextStage() 118 | 119 | ctx, cancel := context.WithCancel(ctx) 120 | defer cancel() 121 | 122 | stage.StartWithContext(func(metricsCtx context.Context) { 123 | defer cancel() // if auxSrv fails to start it signals the whole program that it should shut down 124 | defer logz.LogStructuredPanic() 125 | auxErr = auxSrv.Run(metricsCtx) 126 | }) 127 | 128 | // Leader election 129 | if a.LeaderElectionOptions.LeaderElect { 130 | a.Logger.Info("Starting leader election", logz.NamespaceName(a.LeaderElectionOptions.ConfigMapNamespace)) 131 | ctx, err = options.DoLeaderElection(ctx, a.Logger, a.Name, a.LeaderElectionOptions, a.MainClient.CoreV1(), recorder) 132 | if err != nil { 133 | return err 134 | } 135 | } 136 | return generic.Run(ctx) 137 | } 138 | 139 | // CancelOnInterrupt calls f when os.Interrupt or SIGTERM is received. 140 | // It ignores subsequent interrupts on purpose - program should exit correctly after the first signal. 141 | func CancelOnInterrupt(ctx context.Context, f context.CancelFunc) { 142 | c := make(chan os.Signal, 1) 143 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 144 | go func() { 145 | select { 146 | case <-ctx.Done(): 147 | case <-c: 148 | f() 149 | } 150 | }() 151 | } 152 | 153 | func NewFromFlags(name string, controllers []ctrl.Constructor, flagset *flag.FlagSet, arguments []string) (*App, error) { 154 | a := App{ 155 | Name: name, 156 | Controllers: controllers, 157 | } 158 | for _, cntrlr := range controllers { 159 | cntrlr.AddFlags(flagset) 160 | } 161 | 162 | flagset.BoolVar(&a.Debug, "debug", false, "Enables pprof and prefetcher dump endpoints") 163 | flagset.StringVar(&a.AuxListenOn, "aux-listen-on", defaultAuxServerAddr, "Auxiliary address to listen on. Used for Prometheus metrics server and pprof endpoint. Empty to disable") 164 | 165 | options.BindLeaderElectionFlags(name, &a.LeaderElectionOptions, flagset) 166 | options.BindGenericNamespacedControllerFlags(&a.GenericNamespacedControllerOptions, flagset) 167 | options.BindRestClientFlags(&a.RestClientOptions, flagset) 168 | options.BindLoggerFlags(&a.LoggerOptions, flagset) 169 | 170 | if err := flagutil.ValidateFlags(flagset, arguments); err != nil { 171 | return nil, err 172 | } 173 | 174 | if err := flagset.Parse(arguments); err != nil { 175 | return nil, err 176 | } 177 | if errs := a.GenericNamespacedControllerOptions.DefaultAndValidate(); len(errs) > 0 { 178 | return nil, errors.NewAggregate(errs) 179 | } 180 | if errs := a.RestClientOptions.DefaultAndValidate(); len(errs) > 0 { 181 | return nil, errors.NewAggregate(errs) 182 | } 183 | 184 | var err error 185 | a.RestConfig, err = options.LoadRestClientConfig(name, a.RestClientOptions) 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | a.Logger = options.LoggerFromOptions(a.LoggerOptions) 191 | 192 | // Clients 193 | a.MainClient, err = kubernetes.NewForConfig(a.RestConfig) 194 | if err != nil { 195 | return nil, err 196 | } 197 | 198 | // Metrics 199 | a.PrometheusRegistry = prometheus.NewPedanticRegistry() 200 | err = a.PrometheusRegistry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) 201 | if err != nil { 202 | return nil, err 203 | } 204 | err = a.PrometheusRegistry.Register(prometheus.NewGoCollector()) 205 | if err != nil { 206 | return nil, err 207 | } 208 | 209 | return &a, nil 210 | } 211 | -------------------------------------------------------------------------------- /process/generic.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/ash2k/stager" 9 | "github.com/atlassian/ctrl" 10 | "github.com/atlassian/ctrl/handlers" 11 | "github.com/atlassian/ctrl/logz" 12 | chimw "github.com/go-chi/chi/middleware" 13 | "github.com/pkg/errors" 14 | "github.com/prometheus/client_golang/prometheus" 15 | "go.uber.org/zap" 16 | "golang.org/x/sync/errgroup" 17 | "k8s.io/apimachinery/pkg/runtime/schema" 18 | "k8s.io/client-go/tools/cache" 19 | "k8s.io/client-go/util/workqueue" 20 | ) 21 | 22 | const ( 23 | // Work queue deduplicates scheduled keys. This is the period it waits for duplicate keys before letting the work 24 | // to be dequeued. 25 | workDeduplicationPeriod = 50 * time.Millisecond 26 | 27 | metricsNamespace = "ctrl" 28 | ) 29 | 30 | type Generic struct { 31 | iter uint32 32 | logger *zap.Logger 33 | queue workQueue 34 | workers uint 35 | Controllers map[schema.GroupVersionKind]Holder 36 | Servers map[schema.GroupVersionKind]ServerHolder 37 | Informers map[schema.GroupVersionKind]cache.SharedIndexInformer 38 | } 39 | 40 | func NewGeneric(config *ctrl.Config, queue workqueue.RateLimitingInterface, workers uint, constructors ...ctrl.Constructor) (*Generic, error) { 41 | controllers := make(map[schema.GroupVersionKind]ctrl.Interface) 42 | servers := make(map[schema.GroupVersionKind]ctrl.Server) 43 | holders := make(map[schema.GroupVersionKind]Holder) 44 | informers := make(map[schema.GroupVersionKind]cache.SharedIndexInformer) 45 | serverHolders := make(map[schema.GroupVersionKind]ServerHolder) 46 | wq := workQueue{ 47 | queue: queue, 48 | workDeduplicationPeriod: workDeduplicationPeriod, 49 | } 50 | for _, constr := range constructors { 51 | descr := constr.Describe() 52 | 53 | readyForWork := make(chan struct{}) 54 | queueGvk := wq.newQueueForGvk(descr.Gvk) 55 | groupKind := descr.Gvk.GroupKind() 56 | controllerLogger := config.Logger 57 | constructorConfig := config 58 | constructorConfig.Logger = controllerLogger 59 | 60 | // Extra api data 61 | requestTime := prometheus.NewHistogramVec( 62 | prometheus.HistogramOpts{ 63 | Namespace: metricsNamespace, 64 | Name: "request_time_seconds", 65 | Help: "Number of seconds each request takes to the controller provided server", 66 | }, 67 | []string{"url", "method", "status", "controller", "groupkind"}, 68 | ) 69 | 70 | var allMetrics []prometheus.Collector 71 | 72 | constructed, err := constr.New( 73 | constructorConfig, 74 | &ctrl.Context{ 75 | ReadyForWork: func() { 76 | close(readyForWork) 77 | }, 78 | Middleware: addMetricsMiddleware(requestTime, config.AppName, groupKind.String()), 79 | Informers: informers, 80 | Controllers: controllers, 81 | WorkQueue: queueGvk, 82 | }, 83 | ) 84 | if err != nil { 85 | return nil, errors.Wrapf(err, "failed to construct controller for GVK %s", descr.Gvk) 86 | } 87 | 88 | if constructed.Interface == nil && constructed.Server == nil { 89 | return nil, errors.Wrapf(err, "failed to construct controller or server for GVK %s", descr.Gvk) 90 | } 91 | 92 | if constructed.Interface != nil { 93 | if _, ok := controllers[descr.Gvk]; ok { 94 | return nil, errors.Errorf("duplicate controller for GVK %s", descr.Gvk) 95 | } 96 | inf, ok := informers[descr.Gvk] 97 | if !ok { 98 | return nil, errors.Errorf("controller for GVK %s should have registered an informer for that GVK", descr.Gvk) 99 | } 100 | inf.AddEventHandler(&handlers.GenericHandler{ 101 | Logger: controllerLogger, 102 | WorkQueue: queueGvk, 103 | Gvk: descr.Gvk, 104 | }) 105 | 106 | controllers[descr.Gvk] = constructed.Interface 107 | 108 | objectProcessTime := prometheus.NewHistogramVec( 109 | prometheus.HistogramOpts{ 110 | Namespace: metricsNamespace, 111 | Name: "process_object_seconds", 112 | Help: "Histogram measuring the time it took to process an object", 113 | }, 114 | []string{"controller", "object_namespace", "object", "groupkind"}, 115 | ) 116 | objectProcessErrors := prometheus.NewCounterVec( 117 | prometheus.CounterOpts{ 118 | Namespace: metricsNamespace, 119 | Name: "process_object_errors_total", 120 | Help: "Records the number of times an error was triggered while processing an object", 121 | }, 122 | []string{"controller", "object_namespace", "object", "groupkind", "external", "retriable"}, 123 | ) 124 | 125 | holders[descr.Gvk] = Holder{ 126 | AppName: config.AppName, 127 | Cntrlr: constructed.Interface, 128 | ReadyForWork: readyForWork, 129 | objectProcessTime: objectProcessTime, 130 | objectProcessErrors: objectProcessErrors, 131 | } 132 | 133 | allMetrics = append(allMetrics, objectProcessTime, objectProcessErrors) 134 | } 135 | 136 | if constructed.Server != nil { 137 | if _, ok := servers[descr.Gvk]; ok { 138 | return nil, errors.Errorf("duplicate server for GVK %s", descr.Gvk) 139 | } 140 | servers[descr.Gvk] = constructed.Server 141 | 142 | serverHolders[descr.Gvk] = ServerHolder{ 143 | AppName: config.AppName, 144 | Server: constructed.Server, 145 | ReadyForWork: readyForWork, 146 | requestTime: requestTime, 147 | } 148 | 149 | allMetrics = append(allMetrics, requestTime) 150 | } 151 | 152 | for _, metric := range allMetrics { 153 | if err := constructorConfig.Registry.Register(metric); err != nil { 154 | return nil, errors.WithStack(err) 155 | } 156 | } 157 | } 158 | 159 | return &Generic{ 160 | logger: config.Logger, 161 | queue: wq, 162 | workers: workers, 163 | Controllers: holders, 164 | Servers: serverHolders, 165 | Informers: informers, 166 | }, nil 167 | } 168 | 169 | func (g *Generic) Run(ctx context.Context) error { 170 | // Stager will perform ordered, graceful shutdown 171 | stgr := stager.New() 172 | defer stgr.Shutdown() 173 | defer g.queue.shutDown() 174 | 175 | // Stage: start all informers then wait on them 176 | stage := stgr.NextStage() 177 | for _, inf := range g.Informers { 178 | inf := inf // capture field into a scoped variable to avoid data race 179 | stage.StartWithChannel(func(stopCh <-chan struct{}) { 180 | defer logz.LogStructuredPanic() 181 | inf.Run(stopCh) 182 | }) 183 | } 184 | g.logger.Info("Waiting for informers to sync") 185 | for _, inf := range g.Informers { 186 | if !cache.WaitForCacheSync(ctx.Done(), inf.HasSynced) { 187 | return ctx.Err() 188 | } 189 | } 190 | g.logger.Info("Informers synced") 191 | 192 | // Stage: start all controllers then wait for them to signal ready for work 193 | stage = stgr.NextStage() 194 | for _, c := range g.Controllers { 195 | c := c // capture field into a scoped variable to avoid data race 196 | stage.StartWithContext(func(ctx context.Context) { 197 | defer logz.LogStructuredPanic() 198 | c.Cntrlr.Run(ctx) 199 | }) 200 | } 201 | for gvk, c := range g.Controllers { 202 | select { 203 | case <-ctx.Done(): 204 | g.logger.Sugar().Infof("Was waiting for the controller for %s to become ready for processing", gvk) 205 | return ctx.Err() 206 | case <-c.ReadyForWork: 207 | } 208 | } 209 | 210 | // Stage: start workers 211 | stage = stgr.NextStage() 212 | for i := uint(0); i < g.workers; i++ { 213 | stage.Start(func() { 214 | defer logz.LogStructuredPanic() 215 | g.worker() 216 | }) 217 | } 218 | 219 | if len(g.Servers) == 0 { 220 | <-ctx.Done() 221 | return ctx.Err() 222 | } 223 | 224 | // Stage: start servers 225 | group, ctx := errgroup.WithContext(ctx) 226 | for _, srv := range g.Servers { 227 | server := srv.Server // capture field into a scoped variable to avoid data race 228 | group.Go(func() error { 229 | return server.Run(ctx) 230 | }) 231 | } 232 | return group.Wait() 233 | } 234 | 235 | func (g *Generic) IsReady() bool { 236 | for _, holder := range g.Controllers { 237 | select { 238 | case <-holder.ReadyForWork: 239 | default: 240 | return false 241 | } 242 | } 243 | for _, holder := range g.Servers { 244 | select { 245 | case <-holder.ReadyForWork: 246 | default: 247 | return false 248 | } 249 | } 250 | return true 251 | } 252 | 253 | func addMetricsMiddleware(requestTime *prometheus.HistogramVec, controller, groupKind string) func(http.Handler) http.Handler { 254 | return func(next http.Handler) http.Handler { 255 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 256 | res := chimw.NewWrapResponseWriter(w, r.ProtoMajor) 257 | t0 := time.Now() 258 | next.ServeHTTP(res, r) 259 | tn := time.Since(t0) 260 | 261 | requestTime.WithLabelValues(r.URL.Path, r.Method, string(res.Status()), controller, groupKind).Observe(tn.Seconds()) 262 | }) 263 | } 264 | } 265 | 266 | type Holder struct { 267 | AppName string 268 | Cntrlr ctrl.Interface 269 | ReadyForWork <-chan struct{} 270 | objectProcessTime *prometheus.HistogramVec 271 | objectProcessErrors *prometheus.CounterVec 272 | } 273 | 274 | type ServerHolder struct { 275 | AppName string 276 | Server ctrl.Server 277 | ReadyForWork <-chan struct{} 278 | requestTime *prometheus.HistogramVec 279 | } 280 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2018 Atlassian Pty Ltd 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= 5 | github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= 6 | github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= 7 | github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 8 | github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 9 | github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= 10 | github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= 11 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 12 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 13 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 14 | github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 15 | github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 16 | github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 17 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 18 | github.com/ash2k/stager v0.0.0-20170622123058-6e9c7b0eacd4 h1:pG7CUDQmAqAxVv4smDHWTtorVUI5B7aOcFDfgqtZuWA= 19 | github.com/ash2k/stager v0.0.0-20170622123058-6e9c7b0eacd4/go.mod h1:20N8GhJtHSLeRJvNhy5D1SnEHni4Xlt6p13JQMHYdDY= 20 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= 21 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 22 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 23 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 25 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 26 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 27 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 28 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= 29 | github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 30 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 31 | github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 32 | github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= 33 | github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 34 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 35 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 36 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 37 | github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= 38 | github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= 39 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 40 | github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= 41 | github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= 42 | github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= 43 | github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= 44 | github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= 45 | github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= 46 | github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= 47 | github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 48 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= 49 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 50 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 51 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= 52 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 53 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 54 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 55 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 56 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 57 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 58 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 59 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 60 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 61 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 62 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 63 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 64 | github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= 65 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 66 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 67 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 68 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 69 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 70 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= 71 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 72 | github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= 73 | github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 74 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 75 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 76 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 77 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 78 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 79 | github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= 80 | github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 81 | github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 82 | github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= 83 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 84 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 85 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 86 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 87 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 88 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 89 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 90 | github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= 91 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 92 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 93 | github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 94 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 95 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 96 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 97 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 98 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 99 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 100 | github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 101 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 102 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 103 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 104 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 105 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 106 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 107 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 108 | github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= 109 | github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 110 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 111 | github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= 112 | github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 113 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 114 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 115 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 116 | github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 117 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 118 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 119 | github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= 120 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 121 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= 122 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 123 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= 124 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 125 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= 126 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 127 | github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= 128 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 129 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 130 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 131 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 132 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 133 | github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= 134 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 135 | github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 136 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 137 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 138 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 139 | go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= 140 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 141 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 142 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 143 | go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= 144 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 145 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 146 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 147 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= 148 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 149 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 150 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 151 | golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 152 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 153 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 154 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 155 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 156 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 157 | golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 158 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 159 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 160 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 161 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 162 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 163 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 164 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 165 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 166 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 167 | golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc h1:gkKoSkUmnU6bpS/VhkuO27bzQeSA51uaEfbOW5dNb68= 168 | golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 169 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 170 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 171 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 172 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 173 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 174 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 175 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 176 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 177 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 178 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 179 | golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 180 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 181 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 182 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 183 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 184 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 185 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 186 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= 187 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 188 | golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 189 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 190 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 191 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 192 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 193 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= 194 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 195 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 196 | golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 197 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 198 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 199 | golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 200 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 201 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 202 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 203 | golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 204 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= 205 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 206 | gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= 207 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 208 | gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= 209 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 210 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 211 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 212 | google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= 213 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 214 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 215 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 216 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 217 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 218 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 219 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 220 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 221 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 222 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 223 | gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= 224 | gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 225 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 226 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 227 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 228 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 229 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 230 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 231 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 232 | k8s.io/api v0.0.0-20191003000013-35e20aa79eb8 h1:cJZ/+fVFIFQDkUewZeim5OhZ8X+h48lMtMsZgQMdxOo= 233 | k8s.io/api v0.0.0-20191003000013-35e20aa79eb8/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= 234 | k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655 h1:CS1tBQz3HOXiseWZu6ZicKX361CZLT97UFnnPx0aqBw= 235 | k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= 236 | k8s.io/client-go v0.0.0-20191003000419-f68efa97b39e h1:S7/BObCpTSVf9vlBdEs5hKNzLCq5FMha6bbmRoDGROU= 237 | k8s.io/client-go v0.0.0-20191003000419-f68efa97b39e/go.mod h1:UBFA5lo8nEOepaxS9koNccX/38rYMI3pa1EA1gaFZNg= 238 | k8s.io/code-generator v0.0.0-20190927045949-f81bca4f5e85 h1:OL3Dz4Iv6Z5t5lk2KnWUaplybsGvQG0ZsQSC4G1sqag= 239 | k8s.io/code-generator v0.0.0-20190927045949-f81bca4f5e85/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= 240 | k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 241 | k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= 242 | k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 243 | k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 244 | k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 245 | k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ= 246 | k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 247 | k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ= 248 | k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= 249 | k8s.io/utils v0.0.0-20190801114015-581e00157fb1 h1:+ySTxfHnfzZb9ys375PXNlLhkJPLKgHajBU0N62BDvE= 250 | k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= 251 | modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= 252 | modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= 253 | modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= 254 | modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= 255 | modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= 256 | sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= 257 | sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= 258 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 259 | --------------------------------------------------------------------------------