├── .gitattributes ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── api-server ├── cmd │ └── main.go ├── endpoints │ ├── converter.go │ ├── podmigration.go │ └── types.go ├── manager.go ├── server.go └── signals.go ├── api └── v1 │ ├── groupversion_info.go │ ├── podmigration_types.go │ └── zz_generated.deepcopy.go ├── binaries.tar.bz2 ├── config ├── certmanager │ ├── certificate.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── crd │ ├── bases │ │ └── podmig.dcn.ssu.ac.kr_podmigrations.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_podmigrations.yaml │ │ ├── k8s_list_map_keys.yaml │ │ └── webhook_in_podmigrations.yaml ├── default │ ├── kustomization.yaml │ ├── manager_auth_proxy_patch.yaml │ ├── manager_webhook_patch.yaml │ └── webhookcainjection_patch.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── podmigration_editor_role.yaml │ ├── podmigration_viewer_role.yaml │ ├── role.yaml │ └── role_binding.yaml ├── samples │ ├── checkpoint-example │ │ ├── 1.yaml │ │ ├── checkpoint_example.yaml │ │ ├── eval.txt │ │ ├── jenkin-restore.yaml │ │ ├── jenkin-service.yaml │ │ ├── jenkin.yaml │ │ ├── redmine-restore.yaml │ │ ├── redmine-service.yaml │ │ ├── redmine.yaml │ │ ├── ruby-restore.yaml │ │ ├── ruby-service.yaml │ │ ├── ruby.yaml │ │ ├── test.sh │ │ └── test2.sh │ ├── migration-example │ │ ├── 1.yaml │ │ ├── 2-service.yaml │ │ ├── 2.yaml │ │ ├── 3.yaml │ │ ├── normal.yaml │ │ ├── test1.yaml │ │ ├── test2.yaml │ │ ├── test3.yaml │ │ └── video.mp4 │ ├── podmig_v1_podmigration.yaml │ └── podmig_v1_restore.yaml └── webhook │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── service.yaml ├── controllers ├── helpers.go ├── podmigration_controller.go └── suite_test.go ├── criu-3.14.tar.bz2 ├── go.mod ├── go.sum ├── gui ├── .browserslistrc ├── .eslintrc.js ├── .gitignore ├── README.md ├── babel.config.js ├── package-lock.json ├── package.json ├── public │ ├── favicon.ico │ └── index.html ├── src │ ├── App.vue │ ├── assets │ │ ├── logo.png │ │ └── logo.svg │ ├── components │ │ ├── PodmigrationCard.vue │ │ ├── SidebarHeader.vue │ │ └── utils │ │ │ ├── iconClassMap.js │ │ │ └── statusColorMap.js │ ├── filters.js │ ├── main.js │ ├── mixins.js │ ├── plugins │ │ └── vuetify.js │ ├── router │ │ └── index.js │ ├── services │ │ ├── mock.js │ │ ├── podmigration.js │ │ └── restClient.js │ ├── store │ │ ├── index.js │ │ └── modules │ │ │ ├── app.js │ │ │ ├── auth.js │ │ │ └── index.js │ └── views │ │ ├── AddSource.vue │ │ ├── Dashboard.vue │ │ ├── SourceDetail.vue │ │ ├── SourceList.vue │ │ └── Support.vue └── vue.config.js ├── hack └── boilerplate.go.txt ├── init-cluster-containerd-CRIU.md ├── kubectl-plugin ├── README.md ├── checkpoint-command │ ├── checkpoint_command.go │ └── kubectl-checkpoint ├── cmd.go └── migrate-command │ ├── kubectl-migrate │ └── migrate_command.go ├── main.go └── podmigration.jpg /.gitattributes: -------------------------------------------------------------------------------- 1 | custom-binaries/ filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Kubernetes Generated files - skip generated files, except for vendored files 17 | 18 | !vendor/**/zz_generated.* 19 | 20 | # editor and IDE paraphernalia 21 | .idea 22 | *.swp 23 | *.swo 24 | *~ 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.13 as builder 3 | 4 | WORKDIR /workspace 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | # cache deps before building and copying source so that we don't need to re-download as much 9 | # and so that source changes don't invalidate our downloaded layer 10 | RUN go mod download 11 | 12 | # Copy the go source 13 | COPY main.go main.go 14 | COPY api/ api/ 15 | COPY controllers/ controllers/ 16 | 17 | # Build 18 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go 19 | 20 | # Use distroless as minimal base image to package the manager binary 21 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 22 | FROM gcr.io/distroless/static:nonroot 23 | WORKDIR / 24 | COPY --from=builder /workspace/manager . 25 | USER nonroot:nonroot 26 | 27 | ENTRYPOINT ["/manager"] 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 SSU-DCN 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= controller:latest 4 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 5 | CRD_OPTIONS ?= "crd:trivialVersions=true" 6 | 7 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 8 | ifeq (,$(shell go env GOBIN)) 9 | GOBIN=$(shell go env GOPATH)/bin 10 | else 11 | GOBIN=$(shell go env GOBIN) 12 | endif 13 | 14 | all: manager 15 | 16 | # Run tests 17 | test: generate fmt vet manifests 18 | go test ./... -coverprofile cover.out 19 | 20 | # Build manager binary 21 | manager: generate fmt vet 22 | go build -o bin/manager main.go 23 | 24 | # Run against the configured Kubernetes cluster in ~/.kube/config 25 | run: generate fmt vet manifests 26 | go run ./main.go 27 | 28 | # Install CRDs into a cluster 29 | install: manifests 30 | kustomize build config/crd | kubectl apply -f - 31 | 32 | # Uninstall CRDs from a cluster 33 | uninstall: manifests 34 | kustomize build config/crd | kubectl delete -f - 35 | 36 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 37 | deploy: manifests 38 | cd config/manager && kustomize edit set image controller=${IMG} 39 | kustomize build config/default | kubectl apply -f - 40 | 41 | # Generate manifests e.g. CRD, RBAC etc. 42 | manifests: controller-gen 43 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases 44 | 45 | # Run go fmt against code 46 | fmt: 47 | go fmt ./... 48 | 49 | # Run go vet against code 50 | vet: 51 | go vet ./... 52 | 53 | # Generate code 54 | generate: controller-gen 55 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 56 | 57 | # Build the docker image 58 | docker-build: test 59 | docker build . -t ${IMG} 60 | 61 | # Push the docker image 62 | docker-push: 63 | docker push ${IMG} 64 | 65 | # find or download controller-gen 66 | # download controller-gen if necessary 67 | controller-gen: 68 | ifeq (, $(shell which controller-gen)) 69 | @{ \ 70 | set -e ;\ 71 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ 72 | cd $$CONTROLLER_GEN_TMP_DIR ;\ 73 | go mod init tmp ;\ 74 | go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5 ;\ 75 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ 76 | } 77 | CONTROLLER_GEN=$(GOBIN)/controller-gen 78 | else 79 | CONTROLLER_GEN=$(shell which controller-gen) 80 | endif 81 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: dcn.ssu.ac.kr 2 | repo: github.com/SSU-DCN/podmigration-operator 3 | resources: 4 | - group: podmig 5 | kind: Podmigration 6 | version: v1 7 | version: "2" 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # podmigration-operator 2 | ## The document to init K8s cluster, which enables Podmigration, can be found at: 3 | - https://github.com/SSU-DCN/podmigration-operator/blob/main/init-cluster-containerd-CRIU.md 4 | 5 | ## How to run: 6 | (Run this command at directory podmigration-operator/) 7 | * To run Podmigration operator, which includes CRD and a custom controller: 8 | ``` 9 | $ sudo snap install kustomize 10 | $ sudo apt-get install gcc 11 | $ make manifests 12 | $ make install 13 | $ make run 14 | ``` 15 | * To run api-server, which enables ```kubectl migrate``` command and GUI: (at podmigration-operator/ directory) 16 | ``` 17 | $ go run ./api-server/cmd/main.go 18 | ``` 19 | * To install ```kubectl migrate/checkpoint``` command, follow the guide at https://github.com/SSU-DCN/podmigration-operator/tree/main/kubectl-plugin 20 | * To run GUI: 21 | ``` 22 | $ cd podmigration-operator/gui 23 | $ npm install 24 | $ npm run serve 25 | ``` 26 | ### Demo video: 27 | 1. Migrate video streaming pod from node to node in single cluster: 28 | - https://www.youtube.com/watch?v=M4Ik7aUKhas&t=1s&ab_channel=Xu%C3%A2nT%C6%B0%E1%BB%9DngV%C5%A9 29 | 2. Migrate video streaming pod from cluster to cluster: 30 | - https://www.youtube.com/watch?v=Bpdlgu0XZqo 31 | - https://drive.google.com/file/d/1AeyJZTRJcayBelvXf-CZwFapoquBpns1/view?usp=sharing 32 | 33 | ## Test live-migrate pod: 34 | * Run/check video-stream application: 35 | ``` 36 | $ cd podmigration-operator/config/samples/migration-example 37 | $ kubectl apply -f 2.yaml 38 | $ kubectl get pods 39 | ``` 40 | #### There are three options to live-migrate a running Pod as following: 41 | 1. Live-migrate video-stream application via api-server: 42 | ``` 43 | $ curl --request POST 'localhost:5000/Podmigrations' --header 'Content-Type: application/json' --data-raw '{"name":"test1", "replicas":1, "action":"live-migration", "sourcePod":"video", "destHost":"worker1"}' 44 | $ curl --request GET 'localhost:5000/Podmigrations' 45 | ``` 46 | 2. Live-migrate video-stream application via kubectl apply: 47 | ``` 48 | $ kubectl apply -f test2.yaml 49 | ``` 50 | 3. Live-migrate video-stream application via kubectl migrate command: 51 | - Note: As default, K8S doesn't have ```kubectl migrate``` and ```kubectl checkpoint``` command. To use this extended kubectl plugin please check the guide at https://github.com/SSU-DCN/podmigration-operator/tree/main/kubectl-plugin 52 | ``` 53 | $ kubectl migrate video worker1 54 | ``` 55 | * To delete: 56 | ``` 57 | $ kubectl delete podmigration test2 58 | $ kubectl delete -f test2.yaml 59 | ``` 60 | ## Note 61 | This operator is controller of Kuberntes Pod migration for Kubernetes. It needs several changes to work such as: kubelet, container-runtime-cri (containerd-cri). The modified vesions of Kuberntes and containerd-cri beside this operator can be found in the following repos: 62 | 63 | * https://github.com/vutuong/kubernetes 64 | 65 | 66 | * https://github.com/vutuong/containerd-cri 67 | 68 | ## References 69 | * https://github.com/kubernetes/kubernetes/issues/3949 70 | 71 | ## Workflow 72 | ![alt text](https://github.com/SSU-DCN/podmigration-operator/blob/main/podmigration.jpg?raw=true) 73 | 74 | -------------------------------------------------------------------------------- /api-server/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | apiserver "github.com/SSU-DCN/podmigration-operator/api-server" 7 | podmigrationv1 "github.com/SSU-DCN/podmigration-operator/api/v1" 8 | 9 | "k8s.io/apimachinery/pkg/runtime" 10 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 11 | ctrl "sigs.k8s.io/controller-runtime" 12 | kubelog "sigs.k8s.io/controller-runtime/pkg/log" 13 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 14 | "sigs.k8s.io/controller-runtime/pkg/runtime/signals" 15 | ) 16 | 17 | var ( 18 | runLog = kubelog.Log.WithName("podmigration-cp").WithName("run") 19 | scheme = runtime.NewScheme() 20 | ) 21 | 22 | func init() { 23 | // Initialize the scheme so that kubernetes dynamic client knows 24 | // how to work with new CRD and native kubernetes types 25 | _ = clientgoscheme.AddToScheme(scheme) 26 | _ = podmigrationv1.AddToScheme(scheme) 27 | } 28 | 29 | func main() { 30 | kubelog.SetLogger(zap.New(zap.UseDevMode(true))) 31 | 32 | mgr, err := apiserver.NewManager(ctrl.GetConfigOrDie(), apiserver.Options{ 33 | Scheme: scheme, 34 | Port: 5000, 35 | AllowedDomains: []string{}, 36 | }) 37 | if err != nil { 38 | runLog.Error(err, "unable to create api-server manager") 39 | os.Exit(1) 40 | } 41 | 42 | runLog.Info("starting api-server manager") 43 | if err := mgr.Start(signals.SetupSignalHandler()); err != nil { 44 | runLog.Error(err, "problem running api-server manager") 45 | os.Exit(1) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /api-server/endpoints/converter.go: -------------------------------------------------------------------------------- 1 | package endpoints 2 | 3 | import ( 4 | v1 "github.com/SSU-DCN/podmigration-operator/api/v1" 5 | ) 6 | 7 | var From = &from{} 8 | 9 | type from struct{} 10 | 11 | func (c *from) Object(pm *v1.Podmigration) *Podmigration { 12 | return &Podmigration{ 13 | Name: pm.Name, 14 | Action: pm.Spec.Action, 15 | Replicas: pm.Spec.Replicas, 16 | SourcePod: pm.Spec.SourcePod, 17 | DestHost: pm.Spec.DestHost, 18 | SnapshotPath: pm.Spec.SnapshotPath, 19 | Selector: pm.Spec.Selector, 20 | Status: &pm.Status, 21 | } 22 | } 23 | 24 | func (c *from) List(list *v1.PodmigrationList) *List { 25 | items := make([]Podmigration, len(list.Items)) 26 | for i, r := range list.Items { 27 | items[i] = *c.Object(&r) 28 | } 29 | return &List{ 30 | Items: items, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /api-server/endpoints/podmigration.go: -------------------------------------------------------------------------------- 1 | package endpoints 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | v1 "github.com/SSU-DCN/podmigration-operator/api/v1" 8 | "github.com/emicklei/go-restful" 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | kubelog "sigs.k8s.io/controller-runtime/pkg/log" 13 | ) 14 | 15 | // todo(TUONG): get namespace from request 16 | var namespace = "default" 17 | 18 | type PodmigrationEndpoint struct { 19 | client client.Client 20 | } 21 | 22 | func NewPodmigrationEndpoint(client client.Client) *PodmigrationEndpoint { 23 | return &PodmigrationEndpoint{client: client} 24 | } 25 | 26 | func (pe *PodmigrationEndpoint) SetupWithWS(ws *restful.WebService) { 27 | ws.Route(ws.GET("Podmigrations").To(pe.list). 28 | Doc("List of Podmigrations"). 29 | Returns(200, "OK", &List{})) 30 | 31 | ws.Route(ws.POST("Podmigrations").To(pe.create). 32 | Doc("Create a new Podmigration"). 33 | Reads(&Podmigration{}). 34 | Returns(200, "OK", &Podmigration{}). 35 | Returns(400, "Bad Request", nil)) 36 | } 37 | 38 | func (pe *PodmigrationEndpoint) list(request *restful.Request, response *restful.Response) { 39 | dl := new(v1.PodmigrationList) 40 | err := pe.client.List(request.Request.Context(), dl, &client.ListOptions{}) 41 | if err != nil { 42 | writeError(response, 404, Error{ 43 | Title: "Error", 44 | Details: fmt.Sprintf("Could not retrieve list: %s", err), 45 | }) 46 | } else { 47 | l := From.List(dl) 48 | if err := response.WriteAsJson(l); err != nil { 49 | writeError(response, 404, Error{ 50 | Title: "Error", 51 | Details: "Could not list resources", 52 | }) 53 | } 54 | } 55 | } 56 | 57 | func (pe *PodmigrationEndpoint) create(request *restful.Request, response *restful.Response) { 58 | pm := new(Podmigration) 59 | err := request.ReadEntity(pm) 60 | pm.Action = strings.ToLower(pm.Action) 61 | labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"podmig": "dcn"}} 62 | pm.Selector = &labelSelector 63 | pm.Template = corev1.PodTemplateSpec{ 64 | Spec: corev1.PodSpec{ 65 | Containers: []corev1.Container{}, 66 | }, 67 | } 68 | // fmt.Println("Calling an action: - %v", pm.Action) 69 | fmt.Println(pm) 70 | if err != nil { 71 | writeError(response, 400, Error{ 72 | Title: "Bad Request", 73 | Details: "Could not read entity", 74 | }) 75 | return 76 | } 77 | 78 | if err := pm.Validate(); err != nil { 79 | writeError(response, 400, Error{ 80 | Title: "Validation error", 81 | Details: err.Error(), 82 | }) 83 | return 84 | } 85 | 86 | // Check whether sourcePod of live-migration is exist or not 87 | var sourcePod *corev1.Pod 88 | // var template corev1.PodTemplateSpec 89 | if pm.SourcePod != "" { 90 | fmt.Println(pm.SourcePod) 91 | var childPods corev1.PodList 92 | if err := pe.client.List(request.Request.Context(), &childPods, client.InNamespace(namespace)); err != nil { 93 | writeError(response, 400, Error{ 94 | Title: "Bad Request", 95 | Details: "Could not find any running pod for migration", 96 | }) 97 | return 98 | } 99 | if len(childPods.Items) > 0 { 100 | for _, pod := range childPods.Items { 101 | if pod.Name == pm.SourcePod && pod.Status.Phase == "Running" { 102 | sourcePod = pod.DeepCopy() 103 | } 104 | } 105 | } 106 | } 107 | 108 | if sourcePod == nil { 109 | writeError(response, 400, Error{ 110 | Title: "Bad Request", 111 | Details: "Could not find sourcePod for migration", 112 | }) 113 | return 114 | } 115 | obj := &v1.Podmigration{ 116 | ObjectMeta: metav1.ObjectMeta{Name: pm.Name, Namespace: "default"}, 117 | Spec: v1.PodmigrationSpec{ 118 | Replicas: pm.Replicas, 119 | SourcePod: pm.SourcePod, 120 | DestHost: pm.DestHost, 121 | Selector: pm.Selector, 122 | Action: pm.Action, 123 | SnapshotPath: pm.SnapshotPath, 124 | Template: pm.Template, 125 | }, 126 | } 127 | err = pe.client.Create(request.Request.Context(), obj, &client.CreateOptions{}) 128 | if err != nil { 129 | writeError(response, 400, Error{ 130 | Title: "Error", 131 | Details: fmt.Sprintf("Could not create object: %s", err), 132 | }) 133 | } else { 134 | d := From.Object(obj) 135 | if err := response.WriteAsJson(d); err != nil { 136 | writeError(response, 422, Error{ 137 | Title: "Error", 138 | Details: "Could not write response", 139 | }) 140 | } 141 | } 142 | } 143 | 144 | func writeError(response *restful.Response, httpStatus int, err Error) { 145 | if err := response.WriteHeaderAndJson(httpStatus, err, "application/json"); err != nil { 146 | kubelog.Log.Error(err, "Could not write the error response") 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /api-server/endpoints/types.go: -------------------------------------------------------------------------------- 1 | package endpoints 2 | 3 | import ( 4 | "errors" 5 | // "fmt" 6 | // "strings" 7 | 8 | v1 "github.com/SSU-DCN/podmigration-operator/api/v1" 9 | "github.com/emicklei/go-restful" 10 | 11 | corev1 "k8s.io/api/core/v1" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | ) 14 | 15 | type Endpoint interface { 16 | SetupWithWS(ws *restful.WebService) 17 | } 18 | 19 | type Podmigration struct { 20 | Name string `json:"name"` 21 | DestHost string `json:"destHost"` 22 | Replicas int `json:"replicas"` 23 | Selector *metav1.LabelSelector `json:"selector"` 24 | Action string `json:"action"` 25 | SnapshotPath string `json:"snapshotPath"` 26 | SourcePod string `json:"sourcePod"` 27 | Template corev1.PodTemplateSpec `json:"template,omitempty"` 28 | Status *v1.PodmigrationStatus `json:"status,omitempty"` 29 | } 30 | 31 | func (pm *Podmigration) Validate() error { 32 | var validated bool 33 | validated = true 34 | //TODO(Tuong): check template is valid or not 35 | // if pm.Template == checkTemplate { 36 | // return error.New("template can't be empty") 37 | // } else { 38 | // validated = true 39 | // } 40 | if validated { 41 | return nil 42 | } 43 | return errors.New("source type validation was not performed, type can only be [WebFolder,S3]") 44 | } 45 | 46 | type List struct { 47 | Items []Podmigration `json:"items"` 48 | } 49 | 50 | type Error struct { 51 | Title string `json:"title"` 52 | Details string `json:"details"` 53 | } 54 | -------------------------------------------------------------------------------- /api-server/manager.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | /* 4 | Now we will create an API Server Manager that will create the K8S client and keep a reference to it. 5 | It will also create a cache that will be used to create a cached K8S client, 6 | initialize the cache properly and in the end handle the termination signals. 7 | */ 8 | 9 | import ( 10 | "time" 11 | 12 | "k8s.io/apimachinery/pkg/runtime" 13 | "k8s.io/client-go/rest" 14 | "sigs.k8s.io/controller-runtime/pkg/cache" 15 | "sigs.k8s.io/controller-runtime/pkg/client" 16 | "sigs.k8s.io/controller-runtime/pkg/client/apiutil" 17 | ) 18 | 19 | var ( 20 | defaultRetryPeriod = 2 * time.Second 21 | ) 22 | 23 | // Options to customize Manager behaviour and pass information 24 | type Options struct { 25 | Scheme *runtime.Scheme 26 | Namespace string 27 | Port int 28 | AllowedDomains []string 29 | } 30 | 31 | type Manager interface { 32 | Start(stop <-chan struct{}) error 33 | } 34 | 35 | type manager struct { 36 | config *rest.Config 37 | client client.Client 38 | server *apiServer 39 | started bool 40 | internalStop <-chan struct{} 41 | internalStopper chan<- struct{} 42 | cache cache.Cache 43 | errSignal *errSignaler 44 | port int 45 | allowedDomains []string 46 | } 47 | 48 | func NewManager(config *rest.Config, options Options) (Manager, error) { 49 | mapper, err := apiutil.NewDynamicRESTMapper(config) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | cc, err := cache.New(config, cache.Options{ 55 | Scheme: options.Scheme, 56 | Mapper: mapper, 57 | Resync: &defaultRetryPeriod, 58 | Namespace: options.Namespace, 59 | }) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | c, err := client.New(config, client.Options{Scheme: options.Scheme, Mapper: mapper}) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | stop := make(chan struct{}) 70 | return &manager{ 71 | config: config, 72 | cache: cc, 73 | client: &client.DelegatingClient{ 74 | Reader: &client.DelegatingReader{ 75 | CacheReader: cc, 76 | ClientReader: c, 77 | }, 78 | Writer: c, 79 | StatusClient: c, 80 | }, 81 | internalStop: stop, 82 | internalStopper: stop, 83 | port: options.Port, 84 | allowedDomains: options.AllowedDomains, 85 | }, nil 86 | } 87 | 88 | func (m *manager) Start(stop <-chan struct{}) error { 89 | defer close(m.internalStopper) 90 | // initialize this here so that we reset the signal channel state on every start 91 | m.errSignal = &errSignaler{errSignal: make(chan struct{})} 92 | m.waitForCache() 93 | 94 | srv, err := newApiServer(m.port, m.allowedDomains, m.client) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | go func() { 100 | if err := srv.Start(m.internalStop); err != nil { 101 | m.errSignal.SignalError(err) 102 | } 103 | }() 104 | select { 105 | case <-stop: 106 | return nil 107 | case <-m.errSignal.GotError(): 108 | // Error starting the cache 109 | return m.errSignal.Error() 110 | } 111 | } 112 | 113 | func (m *manager) waitForCache() { 114 | if m.started { 115 | return 116 | } 117 | 118 | go func() { 119 | if err := m.cache.Start(m.internalStop); err != nil { 120 | m.errSignal.SignalError(err) 121 | } 122 | }() 123 | 124 | // Wait for the caches to sync. 125 | m.cache.WaitForCacheSync(m.internalStop) 126 | m.started = true 127 | } 128 | -------------------------------------------------------------------------------- /api-server/server.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | 9 | "github.com/SSU-DCN/podmigration-operator/api-server/endpoints" 10 | "github.com/emicklei/go-restful" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | kubelog "sigs.k8s.io/controller-runtime/pkg/log" 13 | ) 14 | 15 | var ( 16 | log = kubelog.Log.WithName("api-server") 17 | ) 18 | 19 | type apiServer struct { 20 | server *http.Server 21 | } 22 | 23 | func (as *apiServer) Address() string { 24 | return as.server.Addr 25 | } 26 | 27 | func init() { 28 | restful.MarshalIndent = func(v interface{}, prefix, indent string) ([]byte, error) { 29 | var buf bytes.Buffer 30 | encoder := restful.NewEncoder(&buf) 31 | encoder.SetIndent(prefix, indent) 32 | if err := encoder.Encode(v); err != nil { 33 | return nil, err 34 | } 35 | return buf.Bytes(), nil 36 | } 37 | } 38 | 39 | func newApiServer(port int, allowedDomains []string, client client.Client) (*apiServer, error) { 40 | container := restful.NewContainer() 41 | srv := &http.Server{ 42 | Addr: fmt.Sprintf(":%d", port), 43 | Handler: container.ServeMux, 44 | } 45 | 46 | cors := restful.CrossOriginResourceSharing{ 47 | ExposeHeaders: []string{restful.HEADER_AccessControlAllowOrigin}, 48 | AllowedDomains: allowedDomains, 49 | Container: container, 50 | } 51 | 52 | ws := new(restful.WebService) 53 | ws. 54 | Path("/"). 55 | Consumes(restful.MIME_JSON). 56 | Produces(restful.MIME_JSON) 57 | 58 | addEndpoints(ws, client) 59 | container.Add(ws) 60 | container.Filter(cors.Filter) 61 | return &apiServer{ 62 | server: srv, 63 | }, nil 64 | } 65 | 66 | func addEndpoints(ws *restful.WebService, client client.Client) { 67 | resources := []endpoints.Endpoint{ 68 | endpoints.NewPodmigrationEndpoint(client), 69 | } 70 | for _, ep := range resources { 71 | ep.SetupWithWS(ws) 72 | } 73 | } 74 | 75 | func (as *apiServer) Start(stop <-chan struct{}) error { 76 | errChan := make(chan error) 77 | go func() { 78 | err := as.server.ListenAndServe() 79 | if err != nil { 80 | switch err { 81 | case http.ErrServerClosed: 82 | log.Info("Shutting down api-server") 83 | default: 84 | log.Error(err, "Could not start an HTTP Server") 85 | errChan <- err 86 | } 87 | } 88 | }() 89 | log.Info("Starting api-server", "interface", "0.0.0.0", "port", as.Address()) 90 | select { 91 | case <-stop: 92 | log.Info("Shutting down api-server") 93 | return as.server.Shutdown(context.Background()) 94 | case err := <-errChan: 95 | return err 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /api-server/signals.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // var ( 8 | // log = kubelog.Log.WithName("api-server") 9 | // ) 10 | 11 | type errSignaler struct { 12 | // errSignal indicates that an error occurred, when closed. It shouldn't 13 | // be written to. 14 | errSignal chan struct{} 15 | // err is the received error 16 | err error 17 | mu sync.Mutex 18 | } 19 | 20 | func (r *errSignaler) SignalError(err error) { 21 | r.mu.Lock() 22 | defer r.mu.Unlock() 23 | if err == nil { 24 | // non-error, ignore 25 | log.Error(nil, "SignalError called without an (with a nil) error, which should never happen, ignoring") 26 | return 27 | } 28 | if r.err != nil { 29 | // we already have an error, don't try again 30 | return 31 | } 32 | // save the error and report it 33 | r.err = err 34 | close(r.errSignal) 35 | } 36 | 37 | func (r *errSignaler) Error() error { 38 | r.mu.Lock() 39 | defer r.mu.Unlock() 40 | return r.err 41 | } 42 | 43 | func (r *errSignaler) GotError() chan struct{} { 44 | r.mu.Lock() 45 | defer r.mu.Unlock() 46 | return r.errSignal 47 | } 48 | -------------------------------------------------------------------------------- /api/v1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1 contains API Schema definitions for the podmig v1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=podmig.dcn.ssu.ac.kr 20 | package v1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "podmig.dcn.ssu.ac.kr", Version: "v1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /api/v1/podmigration_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 25 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 26 | 27 | // PodmigrationSpec defines the desired state of Podmigration 28 | type PodmigrationSpec struct { 29 | 30 | // Number of desired pods. This is a pointer to distinguish between explicit 31 | // zero and not specified. Defaults to 1. 32 | // +optional 33 | Replicas int `json:"replicas,omitempty"` 34 | SourcePod string `json:"sourcePod,omitempty"` 35 | DestHost string `json:"destHost,omitempty"` 36 | SnapshotPath string `json:"snapshotPath,omitempty"` 37 | 38 | // Label selector for pods. Existing ReplicaSets whose pods are 39 | // selected by this will be the ones affected by this deployment. 40 | // It must match the pod template's labels. 41 | Selector *metav1.LabelSelector `json:"selector"` 42 | Template corev1.PodTemplateSpec `json:"template,omitempty"` 43 | 44 | // Template describes the pods that will be created. 45 | // +kubebuilder:validation:Required 46 | Action string `json:"action"` 47 | // ExcludeNode indicates a node that the Pod should not get scheduled on or get migrated 48 | // away from. 49 | // +kubebuilder:validation:Optional 50 | // ExcludeNodeSelector map[string]string `json:"excludeNodeSelector"` 51 | } 52 | 53 | // PodmigrationStatus defines the observed state of Podmigration 54 | type PodmigrationStatus struct { 55 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 56 | // Important: Run "make" to regenerate code after modifying this file 57 | // State indicates the state of the MigratingPod 58 | State string `json:"state"` 59 | 60 | // CurrentRevision indicates the version of the MigratingPod to generate the current Pod 61 | CurrentRevision string `json:"currentRevision"` 62 | 63 | // ActivePod 64 | ActivePod string `json:"activePod"` 65 | } 66 | 67 | // +kubebuilder:object:root=true 68 | 69 | // Podmigration is the Schema for the podmigrations API 70 | type Podmigration struct { 71 | metav1.TypeMeta `json:",inline"` 72 | metav1.ObjectMeta `json:"metadata,omitempty"` 73 | 74 | Spec PodmigrationSpec `json:"spec,omitempty"` 75 | Status PodmigrationStatus `json:"status,omitempty"` 76 | } 77 | 78 | // +kubebuilder:object:root=true 79 | 80 | // PodmigrationList contains a list of Podmigration 81 | type PodmigrationList struct { 82 | metav1.TypeMeta `json:",inline"` 83 | metav1.ListMeta `json:"metadata,omitempty"` 84 | Items []Podmigration `json:"items"` 85 | } 86 | 87 | func init() { 88 | SchemeBuilder.Register(&Podmigration{}, &PodmigrationList{}) 89 | } 90 | -------------------------------------------------------------------------------- /api/v1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by controller-gen. DO NOT EDIT. 20 | 21 | package v1 22 | 23 | import ( 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | ) 27 | 28 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 29 | func (in *Podmigration) DeepCopyInto(out *Podmigration) { 30 | *out = *in 31 | out.TypeMeta = in.TypeMeta 32 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 33 | in.Spec.DeepCopyInto(&out.Spec) 34 | out.Status = in.Status 35 | } 36 | 37 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Podmigration. 38 | func (in *Podmigration) DeepCopy() *Podmigration { 39 | if in == nil { 40 | return nil 41 | } 42 | out := new(Podmigration) 43 | in.DeepCopyInto(out) 44 | return out 45 | } 46 | 47 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 48 | func (in *Podmigration) DeepCopyObject() runtime.Object { 49 | if c := in.DeepCopy(); c != nil { 50 | return c 51 | } 52 | return nil 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *PodmigrationList) DeepCopyInto(out *PodmigrationList) { 57 | *out = *in 58 | out.TypeMeta = in.TypeMeta 59 | in.ListMeta.DeepCopyInto(&out.ListMeta) 60 | if in.Items != nil { 61 | in, out := &in.Items, &out.Items 62 | *out = make([]Podmigration, len(*in)) 63 | for i := range *in { 64 | (*in)[i].DeepCopyInto(&(*out)[i]) 65 | } 66 | } 67 | } 68 | 69 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodmigrationList. 70 | func (in *PodmigrationList) DeepCopy() *PodmigrationList { 71 | if in == nil { 72 | return nil 73 | } 74 | out := new(PodmigrationList) 75 | in.DeepCopyInto(out) 76 | return out 77 | } 78 | 79 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 80 | func (in *PodmigrationList) DeepCopyObject() runtime.Object { 81 | if c := in.DeepCopy(); c != nil { 82 | return c 83 | } 84 | return nil 85 | } 86 | 87 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 88 | func (in *PodmigrationSpec) DeepCopyInto(out *PodmigrationSpec) { 89 | *out = *in 90 | if in.Selector != nil { 91 | in, out := &in.Selector, &out.Selector 92 | *out = new(metav1.LabelSelector) 93 | (*in).DeepCopyInto(*out) 94 | } 95 | in.Template.DeepCopyInto(&out.Template) 96 | } 97 | 98 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodmigrationSpec. 99 | func (in *PodmigrationSpec) DeepCopy() *PodmigrationSpec { 100 | if in == nil { 101 | return nil 102 | } 103 | out := new(PodmigrationSpec) 104 | in.DeepCopyInto(out) 105 | return out 106 | } 107 | 108 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 109 | func (in *PodmigrationStatus) DeepCopyInto(out *PodmigrationStatus) { 110 | *out = *in 111 | } 112 | 113 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodmigrationStatus. 114 | func (in *PodmigrationStatus) DeepCopy() *PodmigrationStatus { 115 | if in == nil { 116 | return nil 117 | } 118 | out := new(PodmigrationStatus) 119 | in.DeepCopyInto(out) 120 | return out 121 | } 122 | -------------------------------------------------------------------------------- /binaries.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/binaries.tar.bz2 -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for 4 | # breaking changes 5 | apiVersion: cert-manager.io/v1alpha2 6 | kind: Issuer 7 | metadata: 8 | name: selfsigned-issuer 9 | namespace: system 10 | spec: 11 | selfSigned: {} 12 | --- 13 | apiVersion: cert-manager.io/v1alpha2 14 | kind: Certificate 15 | metadata: 16 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 17 | namespace: system 18 | spec: 19 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 20 | dnsNames: 21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 22 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 23 | issuerRef: 24 | kind: Issuer 25 | name: selfsigned-issuer 26 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 27 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/podmig.dcn.ssu.ac.kr_podmigrations.yaml 6 | # +kubebuilder:scaffold:crdkustomizeresource 7 | 8 | patchesStrategicMerge: 9 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 10 | # patches here are for enabling the conversion webhook for each CRD 11 | #- patches/webhook_in_podmigrations.yaml 12 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 13 | 14 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 15 | # patches here are for enabling the CA injection for each CRD 16 | #- patches/cainjection_in_podmigrations.yaml 17 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 18 | patchesJson6902: 19 | - target: 20 | group: apiextensions.k8s.io 21 | version: v1beta1 22 | kind: CustomResourceDefinition 23 | name: podmigrations.podmig.dcn.ssu.ac.kr 24 | path: patches/k8s_list_map_keys.yaml 25 | 26 | # the following config is for teaching kustomize how to do kustomization for CRDs. 27 | configurations: 28 | - kustomizeconfig.yaml 29 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | group: apiextensions.k8s.io 8 | path: spec/conversion/webhookClientConfig/service/name 9 | 10 | namespace: 11 | - kind: CustomResourceDefinition 12 | group: apiextensions.k8s.io 13 | path: spec/conversion/webhookClientConfig/service/namespace 14 | create: false 15 | 16 | varReference: 17 | - path: metadata/annotations 18 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_podmigrations.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: podmigrations.podmig.dcn.ssu.ac.kr 9 | -------------------------------------------------------------------------------- /config/crd/patches/k8s_list_map_keys.yaml: -------------------------------------------------------------------------------- 1 | - op: replace 2 | path: /spec/validation/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/initContainers/items/properties/ports/items/required 3 | value: 4 | - containerPort 5 | - protocol 6 | 7 | - op: replace 8 | path: /spec/validation/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/items/properties/ports/items/required 9 | value: 10 | - containerPort 11 | - protocol -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_podmigrations.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: podmigrations.podmig.dcn.ssu.ac.kr 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: podmigration-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: podmigration-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | #- ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | #- ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | patchesStrategicMerge: 28 | # Protect the /metrics endpoint by putting it behind auth. 29 | # If you want your controller-manager to expose the /metrics 30 | # endpoint w/o any authn/z, please comment the following line. 31 | - manager_auth_proxy_patch.yaml 32 | 33 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 34 | # crd/kustomization.yaml 35 | #- manager_webhook_patch.yaml 36 | 37 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 38 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 39 | # 'CERTMANAGER' needs to be enabled to use ca injection 40 | #- webhookcainjection_patch.yaml 41 | 42 | # the following config is for teaching kustomize how to do var substitution 43 | vars: 44 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 45 | #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 46 | # objref: 47 | # kind: Certificate 48 | # group: cert-manager.io 49 | # version: v1alpha2 50 | # name: serving-cert # this name should match the one in certificate.yaml 51 | # fieldref: 52 | # fieldpath: metadata.namespace 53 | #- name: CERTIFICATE_NAME 54 | # objref: 55 | # kind: Certificate 56 | # group: cert-manager.io 57 | # version: v1alpha2 58 | # name: serving-cert # this name should match the one in certificate.yaml 59 | #- name: SERVICE_NAMESPACE # namespace of the service 60 | # objref: 61 | # kind: Service 62 | # version: v1 63 | # name: webhook-service 64 | # fieldref: 65 | # fieldpath: metadata.namespace 66 | #- name: SERVICE_NAME 67 | # objref: 68 | # kind: Service 69 | # version: v1 70 | # name: webhook-service 71 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | name: https 22 | - name: manager 23 | args: 24 | - "--metrics-addr=127.0.0.1:8080" 25 | - "--enable-leader-election" 26 | -------------------------------------------------------------------------------- /config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | ports: 12 | - containerPort: 9443 13 | name: webhook-server 14 | protocol: TCP 15 | volumeMounts: 16 | - mountPath: /tmp/k8s-webhook-server/serving-certs 17 | name: cert 18 | readOnly: true 19 | volumes: 20 | - name: cert 21 | secret: 22 | defaultMode: 420 23 | secretName: webhook-server-cert 24 | -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | apiVersion: admissionregistration.k8s.io/v1beta1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | name: mutating-webhook-configuration 7 | annotations: 8 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 9 | --- 10 | apiVersion: admissionregistration.k8s.io/v1beta1 11 | kind: ValidatingWebhookConfiguration 12 | metadata: 13 | name: validating-webhook-configuration 14 | annotations: 15 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 16 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | labels: 23 | control-plane: controller-manager 24 | spec: 25 | containers: 26 | - command: 27 | - /manager 28 | args: 29 | - --enable-leader-election 30 | image: controller:latest 31 | name: manager 32 | resources: 33 | limits: 34 | cpu: 100m 35 | memory: 30Mi 36 | requests: 37 | cpu: 100m 38 | memory: 20Mi 39 | terminationGracePeriodSeconds: 10 40 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | selector: 15 | matchLabels: 16 | control-plane: controller-manager 17 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: ["/metrics"] 7 | verbs: ["get"] 8 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: ["authentication.k8s.io"] 7 | resources: 8 | - tokenreviews 9 | verbs: ["create"] 10 | - apiGroups: ["authorization.k8s.io"] 11 | resources: 12 | - subjectaccessreviews 13 | verbs: ["create"] 14 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - role.yaml 3 | - role_binding.yaml 4 | - leader_election_role.yaml 5 | - leader_election_role_binding.yaml 6 | # Comment the following 4 lines if you want to disable 7 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 8 | # which protects your /metrics endpoint. 9 | - auth_proxy_service.yaml 10 | - auth_proxy_role.yaml 11 | - auth_proxy_role_binding.yaml 12 | - auth_proxy_client_clusterrole.yaml 13 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - configmaps/status 23 | verbs: 24 | - get 25 | - update 26 | - patch 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - events 31 | verbs: 32 | - create 33 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/podmigration_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit podmigrations. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: podmigration-editor-role 6 | rules: 7 | - apiGroups: 8 | - podmig.dcn.ssu.ac.kr 9 | resources: 10 | - podmigrations 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - podmig.dcn.ssu.ac.kr 21 | resources: 22 | - podmigrations/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/podmigration_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view podmigrations. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: podmigration-viewer-role 6 | rules: 7 | - apiGroups: 8 | - podmig.dcn.ssu.ac.kr 9 | resources: 10 | - podmigrations 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - podmig.dcn.ssu.ac.kr 17 | resources: 18 | - podmigrations/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - apiGroups: 10 | - podmig.dcn.ssu.ac.kr 11 | resources: 12 | - podmigrations 13 | verbs: 14 | - create 15 | - delete 16 | - get 17 | - list 18 | - patch 19 | - update 20 | - watch 21 | - apiGroups: 22 | - podmig.dcn.ssu.ac.kr 23 | resources: 24 | - podmigrations/status 25 | verbs: 26 | - get 27 | - patch 28 | - update 29 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: simple 5 | labels: 6 | name: simple 7 | #annotations: 8 | #snapshotPolicy: "checkpoint" 9 | #snapshotPath: "/var/lib/kubelet/migration/abc" 10 | spec: 11 | containers: 12 | - name: count 13 | image: alpine 14 | # imagePullPolicy: IfNotPresent 15 | command: ["/bin/ash", "-c", "i=1; while true; do echo $i; i=$((i+1)); sleep 1; done"] 16 | ports: 17 | - containerPort: 80 18 | resources: 19 | limits: 20 | memory: "128Mi" 21 | cpu: "600m" 22 | nodeSelector: 23 | kubernetes.io/hostname: worker2 24 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/checkpoint_example.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: podmig.dcn.ssu.ac.kr/v1 2 | # kind: Podmigration 3 | # metadata: 4 | # name: podmigration-sample 5 | # spec: 6 | # # Add fields here 7 | # foo: bar 8 | apiVersion: podmig.dcn.ssu.ac.kr/v1 9 | kind: Podmigration 10 | metadata: 11 | name: test 12 | labels: 13 | name: test 14 | spec: 15 | replicas: 1 16 | action: checkpoint 17 | # sourcePod is name of a running Pod in the system 18 | sourcePod: simple 19 | # snapshotPath is a path where the checkpoint information will be store 20 | snapshotPath: /var/lib/kubelet/migration/ctconf2020 21 | selector: 22 | podmig: dcn 23 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/eval.txt: -------------------------------------------------------------------------------- 1 | # RUBY application 2 | dcn@dcn:~/podmigration-operator/config/samples/checkpoint-example$ bash test2.sh 3 | pod/red created 4 | 5 | real 0m5.245s 6 | user 0m0.121s 7 | sys 0m0.085s 8 | pod "red" deleted 9 | dcn@dcn:~/podmigration-operator/config/samples/checkpoint-example$ kubectl get pods 10 | No resources found in default namespace. 11 | dcn@dcn:~/podmigration-operator/config/samples/checkpoint-example$ bash test.sh 12 | pod/tuongvx created 13 | 14 | real 0m40.275s 15 | user 0m23.235s 16 | sys 0m9.541s 17 | 18 | # Redmine application 19 | dcn@dcn:~/podmigration-operator/config/samples/checkpoint-example$ bash test.sh 20 | pod/tuongvx created 21 | 22 | real 0m12.536s 23 | user 0m4.649s 24 | sys 0m1.916s 25 | Operation cannot be fulfilled on pods "tuongvx": the object has been modified; please apply your changes to the latest version and try again 26 | pod "tuongvx" deleted 27 | 28 | dcn@dcn:~/podmigration-operator/config/samples/checkpoint-example$ bash test2.sh 29 | pod/red created 30 | 31 | real 0m6.345s 32 | user 0m0.146s 33 | sys 0m0.072s 34 | pod "red" deleted 35 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/jenkin-restore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: jenkins-restore 5 | labels: 6 | app: redmineapp 7 | annotations: 8 | snapshotPolicy: "restore" 9 | snapshotPath: "/var/lib/kubelet/migration/xxx/tuongvx" 10 | spec: 11 | containers: 12 | - name: tuong 13 | image: jenkins/jenkins:lts 14 | ports: 15 | - containerPort: 8080 16 | protocol: TCP 17 | - containerPort: 50000 18 | volumeMounts: 19 | - name: nfs-tmp 20 | mountPath: /tmp 21 | nodeSelector: 22 | kubernetes.io/hostname: worker1 23 | volumes: 24 | - name: nfs-tmp 25 | hostPath: 26 | path: /var/lib/kubelet/migration/jenkins -------------------------------------------------------------------------------- /config/samples/checkpoint-example/jenkin-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: jenkins-service 5 | spec: 6 | type: NodePort 7 | selector: 8 | app: redmineapp 9 | ports: 10 | - port: 8080 11 | targetPort: 8080 12 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/jenkin.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: tuongvx 5 | labels: 6 | app: redmineapp 7 | #annotations: 8 | #snapshotPolicy: "checkpoint" 9 | #snapshotPath: "/var/lib/kubelet/migration/abc" 10 | spec: 11 | containers: 12 | - name: tuong 13 | image: jenkins/jenkins:lts 14 | ports: 15 | - containerPort: 8080 16 | protocol: TCP 17 | - containerPort: 50000 18 | volumeMounts: 19 | - name: nfs-tmp 20 | mountPath: /tmp 21 | nodeSelector: 22 | kubernetes.io/hostname: worker1 23 | volumes: 24 | - name: nfs-tmp 25 | hostPath: 26 | path: /var/lib/kubelet/migration/jenkins -------------------------------------------------------------------------------- /config/samples/checkpoint-example/redmine-restore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: red 5 | labels: 6 | app: redmineapp 7 | annotations: 8 | snapshotPolicy: "restore" 9 | snapshotPath: "/var/lib/kubelet/migration/ooo/tuongvx" 10 | spec: 11 | containers: 12 | - name: tuong 13 | image: tuongvx/redmine 14 | ports: 15 | - containerPort: 3000 16 | protocol: TCP 17 | # volumeMounts: 18 | # # name must match the volume name below 19 | # - name: nfs-log 20 | # mountPath: /usr/src/redmine/log 21 | # - name: nfs-files 22 | # mountPath: /usr/src/redmine/files 23 | # - name: nfs-sqlite 24 | # mountPath: /usr/src/redmine/sqlite 25 | # volumes: 26 | # - name: nfs-log 27 | # hostPath: 28 | # path: /var/lib/kubelet/migration/redmine/log 29 | # type: Directory 30 | # - name: nfs-files 31 | # hostPath: 32 | # path: /var/lib/kubelet/migration/redmine/files 33 | # type: Directory 34 | # - name: nfs-sqlite 35 | # hostPath: 36 | # path: /var/lib/kubelet/migration/redmine/sqlite 37 | nodeSelector: 38 | kubernetes.io/hostname: worker1 39 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/redmine-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redmine-service 5 | spec: 6 | type: NodePort 7 | selector: 8 | app: redmineapp 9 | ports: 10 | - port: 3000 11 | targetPort: 3000 12 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/redmine.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: tuongvx 5 | labels: 6 | app: redmineapp 7 | #annotations: 8 | #snapshotPolicy: "checkpoint" 9 | #snapshotPath: "/var/lib/kubelet/migration/abc" 10 | spec: 11 | containers: 12 | - name: tuong 13 | image: tuongvx/redmine 14 | imagePullPolicy: Always 15 | ports: 16 | - containerPort: 3000 17 | protocol: TCP 18 | volumeMounts: 19 | # name must match the volume name below 20 | - name: nfs-log 21 | mountPath: /usr/src/redmine/log 22 | - name: nfs-files 23 | mountPath: /usr/src/redmine/files 24 | # - name: nfs-publib 25 | # mountPath: /usr/src/redmine/publib 26 | # - name: nfs-plugins 27 | # mountPath: /usr/src/redmine/plugins 28 | # - name: nfs-public 29 | # mountPath: /usr/src/redmine/public 30 | - name: nfs-sqlite 31 | mountPath: /usr/src/redmine/sqlite 32 | volumes: 33 | - name: nfs-log 34 | hostPath: 35 | path: /var/lib/kubelet/migration/redmine/log 36 | type: Directory 37 | - name: nfs-files 38 | hostPath: 39 | path: /var/lib/kubelet/migration/redmine/files 40 | type: Directory 41 | # - name: nfs-publib 42 | # hostPath: 43 | # path: /var/lib/kubelet/migration/redmine/publib 44 | # type: Directory 45 | # - name: nfs-plugins 46 | # hostPath: 47 | # path: /var/lib/kubelet/migration/redmine/plugins 48 | # type: Directory 49 | # - name: nfs-public 50 | # hostPath: 51 | # path: /var/lib/kubelet/migration/redmine/public 52 | # type: Directory 53 | - name: nfs-sqlite 54 | hostPath: 55 | path: /var/lib/kubelet/migration/redmine/sqlite 56 | nodeSelector: 57 | kubernetes.io/hostname: worker1 58 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/ruby-restore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: red 5 | labels: 6 | app: rubyapp 7 | annotations: 8 | snapshotPolicy: "restore" 9 | snapshotPath: "/var/lib/kubelet/migration/fff/tuongvx" 10 | spec: 11 | containers: 12 | - name: tuong 13 | image: tylerfowler/superset 14 | ports: 15 | - containerPort: 8088 16 | protocol: TCP 17 | nodeSelector: 18 | kubernetes.io/hostname: worker1 -------------------------------------------------------------------------------- /config/samples/checkpoint-example/ruby-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: ruby-service 5 | spec: 6 | type: NodePort 7 | selector: 8 | app: rubyapp 9 | ports: 10 | - port: 8088 11 | targetPort: 8088 -------------------------------------------------------------------------------- /config/samples/checkpoint-example/ruby.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: tuongvx 5 | labels: 6 | app: rubyapp 7 | #annotations: 8 | #snapshotPolicy: "checkpoint" 9 | #snapshotPath: "/var/lib/kubelet/migration/abc" 10 | spec: 11 | containers: 12 | - name: tuong 13 | image: tylerfowler/superset 14 | ports: 15 | - containerPort: 8088 16 | protocol: TCP 17 | nodeSelector: 18 | kubernetes.io/hostname: worker1 -------------------------------------------------------------------------------- /config/samples/checkpoint-example/test.sh: -------------------------------------------------------------------------------- 1 | # # test1 with Redmine 2 | kubectl apply -f redmine.yaml 3 | time (while ! curl http://192.168.10.13:31764/ > /dev/null 2>&1; do : ; done ) 4 | kubectl checkpoint tuongvx /var/lib/kubelet/migration/ooo 5 | kubectl delete -f redmine.yaml 6 | 7 | # # test with Web Rail application 8 | # kubectl apply -f ruby.yaml 9 | # time (while ! curl http://192.168.10.13:30087/ > /dev/null 2>&1; do : ; done ) > eval.txt 10 | # kubectl checkpoint tuongvx /var/lib/kubelet/migration/fff 11 | # kubectl delete -f ruby.yaml 12 | 13 | #kubectl apply -f video.yaml 14 | #time (while ! curl http://192.168.10.13:31764/ > /dev/null 2>&1; do : ; done ) 15 | #kubectl checkpoint tuongvx /var/lib/kubelet/migration/ooo 16 | #kubectl delete -f redmine.yaml 17 | -------------------------------------------------------------------------------- /config/samples/checkpoint-example/test2.sh: -------------------------------------------------------------------------------- 1 | # Test1: with Redmine 2 | kubectl apply -f redmine-restore.yaml 3 | time (while ! curl http://192.168.10.13:31764/ > /dev/null 2>&1; do : ; done ) 4 | kubectl delete -f redmine-restore.yaml 5 | 6 | # Test1: with Redmine 7 | # kubectl apply -f ruby-restore.yaml 8 | # time (while ! curl http://192.168.10.13:30087/ > /dev/null 2>&1; do : ; done ) 9 | # kubectl delete -f ruby-restore.yaml 10 | -------------------------------------------------------------------------------- /config/samples/migration-example/1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: simple 5 | labels: 6 | name: simple 7 | #annotations: 8 | #snapshotPolicy: "checkpoint" 9 | #snapshotPath: "/var/lib/kubelet/migration/abc" 10 | spec: 11 | containers: 12 | - name: count 13 | image: alpine 14 | # imagePullPolicy: IfNotPresent 15 | command: ["/bin/ash", "-c", "i=1; while true; do echo $i; i=$((i+1)); sleep 1; done"] 16 | ports: 17 | - containerPort: 80 18 | resources: 19 | limits: 20 | memory: "128Mi" 21 | cpu: "600m" 22 | nodeSelector: 23 | kubernetes.io/hostname: worker1 24 | -------------------------------------------------------------------------------- /config/samples/migration-example/2-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: video-service 5 | spec: 6 | type: NodePort 7 | selector: 8 | app: video 9 | ports: 10 | - port: 8080 11 | targetPort: 8080 12 | -------------------------------------------------------------------------------- /config/samples/migration-example/2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: video 5 | labels: 6 | app: video 7 | #annotations: 8 | #snapshotPolicy: "checkpoint" 9 | #snapshotPath: "/var/lib/kubelet/migration/abc" 10 | spec: 11 | containers: 12 | - name: vlc 13 | image: tuongvx/vlc-app:latest 14 | ports: 15 | - containerPort: 8080 16 | protocol: TCP 17 | #args: ["big_buck_bunny.mp4","--loop","--sout", "#transcode{scodec=none}:http{mux=ffmpeg{mux=flv},dst=:8080/}"] 18 | args: ["frozen.mp4","--loop","--sout", "#transcode{scodec=none}:http{mux=ffmpeg{mux=flv},dst=:8080/}"] 19 | nodeSelector: 20 | kubernetes.io/hostname: worker1 21 | -------------------------------------------------------------------------------- /config/samples/migration-example/3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: migration 5 | labels: 6 | app: video 7 | annotations: 8 | snapshotPolicy: "restore" 9 | snapshotPath: "/var/lib/kubelet/migration/ooo/video" 10 | spec: 11 | containers: 12 | - name: vlc 13 | image: tuongvx/vlc-app:latest 14 | ports: 15 | - containerPort: 8080 16 | protocol: TCP 17 | #args: ["big_buck_bunny.mp4","--loop","--sout", "#transcode{scodec=none}:http{mux=ffmpeg{mux=flv},dst=:8080/}"] 18 | args: ["frozen.mp4","--loop","--sout", "#transcode{scodec=none}:http{mux=ffmpeg{mux=flv},dst=:8080/}"] 19 | nodeSelector: 20 | kubernetes.io/hostname: worker2 21 | -------------------------------------------------------------------------------- /config/samples/migration-example/normal.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/config/samples/migration-example/normal.yaml -------------------------------------------------------------------------------- /config/samples/migration-example/test1.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: podmig.dcn.ssu.ac.kr/v1 2 | # kind: Podmigration 3 | # metadata: 4 | # name: podmigration-sample 5 | # spec: 6 | # # Add fields here 7 | # foo: bar 8 | apiVersion: podmig.dcn.ssu.ac.kr/v1 9 | kind: Podmigration 10 | metadata: 11 | name: test1 12 | labels: 13 | name: test1 14 | spec: 15 | replicas: 1 16 | action: live-migration 17 | # snapshotPath: /var/lib/kubelet/migration 18 | sourcePod: simple 19 | destHost: worker2 20 | selector: 21 | podmig: dcn 22 | # template: 23 | # metadata: 24 | # labels: 25 | # app: simple 26 | # spec: 27 | # containers: 28 | # - name: count 29 | # image: alpine 30 | # ports: 31 | # - containerPort: 80 32 | # resources: 33 | # limits: 34 | # memory: "128Mi" 35 | # cpu: "600m" 36 | # nodeSelector: 37 | # kubernetes.io/hostname: worker1 38 | -------------------------------------------------------------------------------- /config/samples/migration-example/test2.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: podmig.dcn.ssu.ac.kr/v1 2 | # kind: Podmigration 3 | # metadata: 4 | # name: podmigration-sample 5 | # spec: 6 | # # Add fields here 7 | # foo: bar 8 | apiVersion: podmig.dcn.ssu.ac.kr/v1 9 | kind: Podmigration 10 | metadata: 11 | name: test2 12 | labels: 13 | # name: test2 14 | app: video 15 | spec: 16 | replicas: 1 17 | action: live-migration 18 | # snapshotPath: /var/lib/kubelet/migration 19 | sourcePod: video 20 | destHost: worker2 21 | selector: 22 | podmig: dcn 23 | # template: 24 | # metadata: 25 | # labels: 26 | # app: video 27 | # spec: 28 | # containers: 29 | # - name: vlc 30 | # image: tuongvx/vlc-app:latest 31 | # ports: 32 | # - containerPort: 8080 33 | # protocol: TCP 34 | # #args: ["big_buck_bunny.mp4","--loop","--sout", "#transcode{scodec=none}:http{mux=ffmpeg{mux=flv},dst=:8080/}"] 35 | # nodeSelector: 36 | # kubernetes.io/hostname: worker2 -------------------------------------------------------------------------------- /config/samples/migration-example/test3.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: podmig.dcn.ssu.ac.kr/v1 2 | # kind: Podmigration 3 | # metadata: 4 | # name: podmigration-sample 5 | # spec: 6 | # # Add fields here 7 | # foo: bar 8 | apiVersion: podmig.dcn.ssu.ac.kr/v1 9 | kind: Podmigration 10 | metadata: 11 | name: test3 12 | labels: 13 | app: video 14 | spec: 15 | replicas: 1 16 | action: "live-migration" 17 | # snapshotPath: /var/lib/kubelet/migration 18 | sourcePod: "video-migration-86" 19 | destHost: worker2 20 | selector: 21 | podmig: dcn 22 | # template: 23 | # metadata: 24 | # labels: 25 | # app: video 26 | # spec: 27 | # containers: 28 | # - name: vlc 29 | # image: tuongvx/vlc-app:latest 30 | # ports: 31 | # - containerPort: 8080 32 | # protocol: TCP 33 | # #args: ["big_buck_bunny.mp4","--loop","--sout", "#transcode{scodec=none}:http{mux=ffmpeg{mux=flv},dst=:8080/}"] 34 | # nodeSelector: 35 | # kubernetes.io/hostname: worker1 -------------------------------------------------------------------------------- /config/samples/migration-example/video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/config/samples/migration-example/video.mp4 -------------------------------------------------------------------------------- /config/samples/podmig_v1_podmigration.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: podmig.dcn.ssu.ac.kr/v1 2 | # kind: Podmigration 3 | # metadata: 4 | # name: podmigration-sample 5 | # spec: 6 | # # Add fields here 7 | # foo: bar 8 | apiVersion: podmig.dcn.ssu.ac.kr/v1 9 | kind: Podmigration 10 | metadata: 11 | name: test 12 | labels: 13 | name: test 14 | spec: 15 | replicas: 1 16 | action: "live-migration" 17 | snapshotPath: /var/lib/kubelet/migration 18 | sourcePod: redis 19 | destHost: worker2 20 | selector: 21 | podmig: dcn -------------------------------------------------------------------------------- /config/samples/podmig_v1_restore.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: podmig.dcn.ssu.ac.kr/v1 2 | # kind: Podmigration 3 | # metadata: 4 | # name: podmigration-sample 5 | # spec: 6 | # # Add fields here 7 | # foo: bar 8 | apiVersion: podmig.dcn.ssu.ac.kr/v1 9 | kind: Podmigration 10 | metadata: 11 | name: test 12 | labels: 13 | name: test 14 | spec: 15 | replicas: 1 16 | action: restore 17 | # snapshotPath is a path which include checkpoint infomation of a pod 18 | # There are two ways create a checkpoint: 19 | # 1. Ref to podmigration-operator/config/samples/checkpoint-example/checkpoint_example.yaml to see how to checkpoint a running pod by given template 20 | # 2. Ref to podmigration-operator/tree/main/kubectl-plugin to see how to checkpoint a running pod by command #kubectl checkpoint 21 | snapshotPath: /var/lib/kubelet/migration/ctconf2020/simple 22 | # sourcePod: simple 23 | destHost: worker2 24 | selector: 25 | podmig: dcn 26 | # When restore a number of pods from existing checkpoint infomation, a pre-template should be defined to pre-create a new pod first, then the checkpoint info will be loaded 27 | template: 28 | metadata: 29 | name: simple 30 | labels: 31 | name: simple 32 | spec: 33 | containers: 34 | - name: count 35 | image: alpine 36 | ports: 37 | - containerPort: 80 38 | protocol: TCP 39 | 40 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 9443 11 | selector: 12 | control-plane: controller-manager 13 | -------------------------------------------------------------------------------- /controllers/helpers.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "fmt" 5 | 6 | podmigv1 "github.com/SSU-DCN/podmigration-operator/api/v1" 7 | appsv1 "k8s.io/api/apps/v1" 8 | corev1 "k8s.io/api/core/v1" 9 | "k8s.io/apimachinery/pkg/api/meta" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/labels" 12 | "k8s.io/apimachinery/pkg/runtime" 13 | ctrl "sigs.k8s.io/controller-runtime" 14 | ) 15 | 16 | func (r *PodmigrationReconciler) desiredPod(migratingPod podmigv1.Podmigration, parentObject runtime.Object, namespace string, template *corev1.PodTemplateSpec) (*corev1.Pod, error) { 17 | // template := &migratingPod.Spec.Template 18 | desiredLabels := getPodsLabelSet(template) 19 | desiredFinalizers := getPodsFinalizers(template) 20 | desiredAnnotations := getPodsAnnotationSet(&migratingPod, template) 21 | accessor, _ := meta.Accessor(parentObject) 22 | prefix := getPodsPrefix(accessor.GetName()) 23 | pod := &corev1.Pod{ 24 | ObjectMeta: metav1.ObjectMeta{ 25 | Namespace: namespace, 26 | Labels: desiredLabels, 27 | Finalizers: desiredFinalizers, 28 | Annotations: desiredAnnotations, 29 | GenerateName: prefix, 30 | }, 31 | } 32 | pod.Spec = *template.Spec.DeepCopy() 33 | if err := ctrl.SetControllerReference(&migratingPod, pod, r.Scheme); err != nil { 34 | return pod, err 35 | } 36 | return pod, nil 37 | } 38 | 39 | func (r *PodmigrationReconciler) desiredDeployment(migratingPod podmigv1.Podmigration, parentObject runtime.Object, namespace string, template *corev1.PodTemplateSpec) (*appsv1.Deployment, error) { 40 | // template := &migratingPod.Spec.Template 41 | desiredLabels := getPodsLabelSet(template) 42 | desiredFinalizers := getPodsFinalizers(template) 43 | desiredAnnotations := getPodsAnnotationSet(&migratingPod, template) 44 | accessor, _ := meta.Accessor(parentObject) 45 | prefix := getPodsPrefix(accessor.GetName()) 46 | podSpec := *template.Spec.DeepCopy() 47 | replicas := int32(migratingPod.Spec.Replicas) 48 | desiredLabels["migratingPod"] = migratingPod.Name 49 | depl := &appsv1.Deployment{ 50 | TypeMeta: metav1.TypeMeta{APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "Deployment"}, 51 | ObjectMeta: metav1.ObjectMeta{ 52 | Name: migratingPod.Name, 53 | Namespace: migratingPod.Namespace, 54 | }, 55 | Spec: appsv1.DeploymentSpec{ 56 | Selector: &metav1.LabelSelector{ 57 | MatchLabels: desiredLabels, 58 | }, 59 | Replicas: &replicas, 60 | Template: corev1.PodTemplateSpec{ 61 | ObjectMeta: metav1.ObjectMeta{ 62 | Namespace: namespace, 63 | Labels: desiredLabels, 64 | Finalizers: desiredFinalizers, 65 | Annotations: desiredAnnotations, 66 | GenerateName: prefix, 67 | }, 68 | Spec: podSpec, 69 | }, 70 | }, 71 | } 72 | if err := ctrl.SetControllerReference(&migratingPod, depl, r.Scheme); err != nil { 73 | return depl, err 74 | } 75 | return depl, nil 76 | } 77 | 78 | func getPodsLabelSet(template *corev1.PodTemplateSpec) labels.Set { 79 | desiredLabels := make(labels.Set) 80 | for k, v := range template.Labels { 81 | desiredLabels[k] = v 82 | } 83 | return desiredLabels 84 | } 85 | 86 | func getPodsFinalizers(template *corev1.PodTemplateSpec) []string { 87 | desiredFinalizers := make([]string, len(template.Finalizers)) 88 | copy(desiredFinalizers, template.Finalizers) 89 | return desiredFinalizers 90 | } 91 | 92 | func getPodsAnnotationSet(migratingPod *podmigv1.Podmigration, template *corev1.PodTemplateSpec) labels.Set { 93 | // template := &migratingPod.Spec.Template 94 | desiredAnnotations := make(labels.Set) 95 | for k, v := range template.Annotations { 96 | desiredAnnotations[k] = v 97 | } 98 | 99 | desiredAnnotations["sourcePod"] = migratingPod.Spec.SourcePod 100 | desiredAnnotations["snapshotPolicy"] = migratingPod.Spec.Action 101 | desiredAnnotations["snapshotPath"] = migratingPod.Spec.SnapshotPath 102 | return desiredAnnotations 103 | } 104 | 105 | func getPodsPrefix(controllerName string) string { 106 | // use the dash (if the name isn't too long) to make the pod name a bit prettier 107 | prefix := fmt.Sprintf("%s-", controllerName) 108 | // if len(validation.ValidatePodName(prefix, true)) != 0 { 109 | // prefix = controllerName 110 | // } 111 | return prefix 112 | } 113 | 114 | // RemoveString removes the element at position i from a string array without preserving 115 | // the order. 116 | // https://stackoverflow.com/a/37335777/4430124 117 | func RemoveString(s []string, i int) []string { 118 | s[i] = s[len(s)-1] 119 | return s[:len(s)-1] 120 | } 121 | -------------------------------------------------------------------------------- /controllers/podmigration_controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "math/rand" 22 | "os" 23 | "os/exec" 24 | "path" 25 | "strconv" 26 | "strings" 27 | "time" 28 | 29 | "github.com/go-logr/logr" 30 | "k8s.io/apimachinery/pkg/runtime" 31 | ctrl "sigs.k8s.io/controller-runtime" 32 | "sigs.k8s.io/controller-runtime/pkg/client" 33 | 34 | podmigv1 "github.com/SSU-DCN/podmigration-operator/api/v1" 35 | appsv1 "k8s.io/api/apps/v1" 36 | 37 | corev1 "k8s.io/api/core/v1" 38 | // corev1 "github.com/vutuong/kubernetes/tree/feature/pod-migration/staging/src/k8s.io/api/core/v1" 39 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 | ) 41 | 42 | const ( 43 | // podOwnerKey = ".metadata.controller" 44 | podOwnerKey = "migratingPod" 45 | // migratingPodFinalizer = "podmig.schrej.net/Migrate" 46 | ) 47 | 48 | // PodmigrationReconciler reconciles a Podmigration object 49 | type PodmigrationReconciler struct { 50 | client.Client 51 | Log logr.Logger 52 | Scheme *runtime.Scheme 53 | } 54 | 55 | // +kubebuilder:rbac:groups=podmig.dcn.ssu.ac.kr,resources=podmigrations,verbs=get;list;watch;create;update;patch;delete 56 | // +kubebuilder:rbac:groups=podmig.dcn.ssu.ac.kr,resources=podmigrations/status,verbs=get;update;patch 57 | 58 | func (r *PodmigrationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { 59 | ctx := context.Background() 60 | log := r.Log.WithValues("podmigration", req.NamespacedName) 61 | 62 | // your logic here 63 | // Load the podMigration resource object, if there is no Object, return directly 64 | var migratingPod podmigv1.Podmigration 65 | if err := r.Get(ctx, req.NamespacedName, &migratingPod); err != nil { 66 | return ctrl.Result{}, client.IgnoreNotFound(err) 67 | } 68 | log.Info("", "print test", migratingPod.Spec) 69 | var template *corev1.PodTemplateSpec 70 | if migratingPod.Spec.Template.ObjectMeta.Name != "" { 71 | template = &migratingPod.Spec.Template 72 | } else { 73 | var Err error 74 | template, Err = r.getSourcePodTemplate(ctx, migratingPod.Spec.SourcePod, req.Namespace) 75 | if Err != nil || template == nil { 76 | log.Error(Err, "sourcePod not exist", "pod", migratingPod.Spec.SourcePod) 77 | return ctrl.Result{}, Err 78 | } 79 | } 80 | 81 | if migratingPod.Spec.DestHost != "" { 82 | template.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": migratingPod.Spec.DestHost} 83 | } 84 | 85 | desiredLabels := getPodsLabelSet(template) 86 | desiredLabels["migratingPod"] = migratingPod.Name 87 | annotations := getPodsAnnotationSet(&migratingPod, template) 88 | 89 | // Then list all pods controlled by the Podmigration resource object 90 | var childPods corev1.PodList 91 | if err := r.List(ctx, &childPods, client.InNamespace(req.Namespace), client.MatchingLabels(desiredLabels)); err != nil { 92 | log.Error(err, "unable to list child pods") 93 | return ctrl.Result{}, err 94 | } 95 | 96 | pod, err := r.desiredPod(migratingPod, &migratingPod, req.Namespace, template) 97 | if err != nil { 98 | return ctrl.Result{}, err 99 | } 100 | 101 | depl, err := r.desiredDeployment(migratingPod, &migratingPod, req.Namespace, template) 102 | if err != nil { 103 | return ctrl.Result{}, err 104 | } 105 | 106 | log.Info("", "annotations ", annotations["snapshotPath"]) 107 | log.Info("", "number of existing pod ", len(childPods.Items)) 108 | log.Info("", "desired pod ", pod) 109 | log.Info("", "number of desired pod ", migratingPod.Spec.Replicas) 110 | 111 | count, _, _ := r.getActualRunningPod(&childPods) 112 | log.Info("", "number of actual running pod ", count) 113 | 114 | if annotations["snapshotPolicy"] == "live-migration" && annotations["sourcePod"] != "" { 115 | // We are live-migrate a running pod here - Hot scale 116 | // Step1: Check source pod is exist or not clean previous source pod checkpoint/restore annotations and snapshotPath 117 | sourcePod, err := r.checkPodExist(ctx, annotations["sourcePod"], req.Namespace) 118 | if err != nil || sourcePod == nil { 119 | log.Error(err, "sourcePod not exist", "pod", annotations["sourcePod"]) 120 | return ctrl.Result{}, err 121 | } 122 | if err := r.removeCheckpointPod(ctx, sourcePod, "/var/lib/kubelet/migration/kkk", "", req.Namespace); err != nil { 123 | log.Error(err, "unable to remove checkpoint", "pod", sourcePod) 124 | return ctrl.Result{}, err 125 | } 126 | log.Info("", "Live-migration", "Step 1 - Check source pod is exist or not - completed") 127 | log.Info("", "sourcePod ok ", sourcePod) 128 | log.Info("", "sourcePod status ", sourcePod.Status.Phase) 129 | // Step2: checkpoint sourcePod 130 | // copySourcePod := sourcePod.DeepCopy() 131 | if err := r.checkpointPod(ctx, sourcePod, ""); err != nil { 132 | log.Error(err, "unable to checkpoint", "pod", sourcePod) 133 | return ctrl.Result{}, err 134 | } 135 | log.Info("", "Live-migration", "Step 2 - checkpoint source Pod - completed") 136 | // TODO(TUONG): make migrate all container inside Pod 137 | // for container := range copySourcePod.Spec.Containers { 138 | // fmt.Println(copySourcePod.Spec.Containers[container].Name) 139 | // log.Info("", "container of pod", copySourcePod.Spec.Containers[container].Name) 140 | // } 141 | 142 | // Step3: wait until checkpoint info are created 143 | container := sourcePod.Spec.Containers[0].Name 144 | checkpointPath := path.Join("/var/lib/kubelet/migration/kkk", strings.Split(sourcePod.Name, "-")[0]) 145 | log.Info("", "live-migration pod", container) 146 | for { 147 | _, err := os.Stat(path.Join(checkpointPath, container, "descriptors.json")) 148 | if os.IsNotExist(err) { 149 | time.Sleep(100 * time.Millisecond) 150 | } else { 151 | break 152 | } 153 | } 154 | log.Info("", "Live-migration", "checkpointPath"+checkpointPath) 155 | log.Info("", "Live-migration", "Step 3 - Wait until checkpoint info are created - completed") 156 | // time.Sleep(10) 157 | // Step4: restore destPod from sourcePod checkpoted info 158 | newPod, err := r.restorePod(ctx, pod, annotations["sourcePod"], checkpointPath) 159 | if err != nil { 160 | log.Error(err, "unable to restore", "pod", sourcePod) 161 | return ctrl.Result{}, err 162 | } 163 | log.Info("", "Live-migration", "Step 4 - Restore destPod from sourcePod's checkpointed info - completed") 164 | // time.Sleep(5) 165 | for { 166 | status, _ := r.checkPodExist(ctx, newPod.Name, req.Namespace) 167 | if status != nil { 168 | log.Info("", "Live-migration", "Step 4.1 - Check whether if newPod is Running or not - completed"+status.Name+string(status.Status.Phase)) 169 | break 170 | } else { 171 | time.Sleep(200 * time.Millisecond) 172 | } 173 | } 174 | log.Info("", "Live-migration", "Step 4.1 - Check whether if newPod is Running or not - completed") 175 | // Step5: Clean checkpointpod process and checkpointPath 176 | // if err := r.removeCheckpointPod(ctx, sourcePod, "/var/lib/kubelet/migration/kkk", newPod.Name, req.Namespace); err != nil { 177 | // log.Error(err, "unable to remove checkpoint", "pod", sourcePod) 178 | // return ctrl.Result{}, err 179 | // } 180 | // log.Info("", "Live-migration", "Step 5 - Clean checkpointPod process and checkpointPath - completed") 181 | 182 | // // Step6: Delete source Pod 183 | if err := r.deletePod(ctx, sourcePod); err != nil { 184 | log.Error(err, "unable to delete", "source pod", sourcePod) 185 | return ctrl.Result{}, err 186 | } 187 | log.Info("", "Live-migration", "Step 6 - Delete the source pod - completed") 188 | return ctrl.Result{}, nil 189 | } 190 | if count == 0 && annotations["snapshotPolicy"] == "restore" { 191 | // We are restoring pods here - Warm scale 192 | _, err := os.Stat(annotations["snapshotPath"]) 193 | if annotations["snapshotPolicy"] != "restore" && os.IsNotExist(err) { 194 | pod.ObjectMeta.Annotations["snapshotPolicy"] = "" 195 | pod.ObjectMeta.Annotations["snapshotPath"] = "" 196 | } 197 | if err := r.createMultiPod(ctx, migratingPod.Spec.Replicas, depl); err != nil { 198 | log.Error(err, "unable to create Pod for restore", "pod", pod) 199 | return ctrl.Result{}, err 200 | } 201 | log.Info("", "Restore", "Step 0 - Create multiple pods from checkpoint infomation - completed") 202 | } else if count != 0 && count != migratingPod.Spec.Replicas { 203 | _, err := os.Stat(annotations["snapshotPath"]) 204 | if annotations["snapshotPolicy"] != "restore" && os.IsNotExist(err) { 205 | pod.ObjectMeta.Annotations["snapshotPolicy"] = "" 206 | pod.ObjectMeta.Annotations["snapshotPath"] = "" 207 | } 208 | if err := r.updateMultiPod(ctx, migratingPod.Spec.Replicas-count, depl); err != nil { 209 | log.Error(err, "unable to create Pod for restore", "pod", pod) 210 | return ctrl.Result{}, err 211 | } 212 | log.Info("", "Restore", "Step 0 - Scale multiple pods from checkpoint infomation - completed") 213 | } else { 214 | // We are checkpointing a running pod here 215 | if annotations["snapshotPolicy"] == "checkpoint" && annotations["sourcePod"] != "" { 216 | _, err := os.Stat(annotations["snapshotPath"]) 217 | // Step1: Check source pod is exist or not 218 | sourcePod, err := r.checkPodExist(ctx, annotations["sourcePod"], req.Namespace) 219 | if err != nil || sourcePod == nil { 220 | log.Error(err, "sourcePod not exist", "pod", annotations["sourcePod"]) 221 | return ctrl.Result{}, err 222 | } 223 | log.Info("", "Checkpoint", "Step 1 - Check the snapshotPaht is exist or not - completed") 224 | // Step2: Clean previous checkpoint folder if exist 225 | if err := r.removeCheckpointPod(ctx, sourcePod, annotations["snapshotPath"], "", req.Namespace); err != nil { 226 | log.Error(err, "unable to remove checkpoint", "pod", sourcePod) 227 | return ctrl.Result{}, err 228 | } 229 | log.Info("", "Checkpoint", "Step 2 - Clean previous checkpoint folder if exist - completed") 230 | // Step3: Checkpoint the source pod now 231 | if err := r.checkpointPod(ctx, sourcePod, annotations["snapshotPath"]); err != nil { 232 | log.Error(err, "unable to checkpoint", "pod", sourcePod) 233 | return ctrl.Result{}, err 234 | } 235 | log.Info("", "Checkpoint", "Step 3 - Checkpoint source Pod and save it - completed") 236 | } 237 | } 238 | return ctrl.Result{}, nil 239 | } 240 | 241 | func (r *PodmigrationReconciler) getActualRunningPod(childPods *corev1.PodList) (int, corev1.PodList, corev1.PodList) { 242 | // if a pod is deleted, remove it from Actual running pod list 243 | count := 0 244 | var actualRunningPod, isDeletingPod corev1.PodList 245 | for _, pod := range childPods.Items { 246 | if !pod.DeletionTimestamp.IsZero() { 247 | isDeletingPod.Items = append(isDeletingPod.Items, pod) 248 | } else { 249 | actualRunningPod.Items = append(actualRunningPod.Items, pod) 250 | count++ 251 | } 252 | } 253 | return count, actualRunningPod, isDeletingPod 254 | } 255 | 256 | func (r *PodmigrationReconciler) createMultiPod(ctx context.Context, replicas int, depl *appsv1.Deployment) error { 257 | if err := r.Create(ctx, depl); err != nil { 258 | return err 259 | } 260 | return nil 261 | } 262 | 263 | func (r *PodmigrationReconciler) updateMultiPod(ctx context.Context, replicas int, depl *appsv1.Deployment) error { 264 | if err := r.Update(ctx, depl); err != nil { 265 | return err 266 | } 267 | return nil 268 | } 269 | 270 | func (r *PodmigrationReconciler) deletePod(ctx context.Context, pod *corev1.Pod) error { 271 | if err := r.Delete(ctx, pod); err != nil { 272 | return err 273 | } 274 | return nil 275 | } 276 | 277 | func (r *PodmigrationReconciler) checkpointPod(ctx context.Context, pod *corev1.Pod, snapshotPath string) error { 278 | snapshotPolicy := "checkpoint" 279 | if snapshotPath == "" { 280 | snapshotPath = "/var/lib/kubelet/migration/kkk" 281 | } 282 | if err := r.updateAnnotations(ctx, pod, snapshotPolicy, snapshotPath); err != nil { 283 | return err 284 | } 285 | return nil 286 | } 287 | 288 | func (r *PodmigrationReconciler) restorePod(ctx context.Context, pod *corev1.Pod, sourcePod, checkpointPath string) (*corev1.Pod, error) { 289 | // targetPod := pod.DeepCopy() 290 | // targetPod.Finalizers = append(targetPod.Finalizers, migratingPodFinalizer) 291 | s1 := rand.NewSource(time.Now().UnixNano()) 292 | number := rand.New(s1) 293 | sourcePod = strings.Split(sourcePod, "-migration-")[0] 294 | pod.Name = sourcePod + "-migration-" + strconv.Itoa(number.Intn(100)) 295 | // pod.Spec.ClonePod = sourcePod 296 | pod.ObjectMeta.Annotations["snapshotPolicy"] = "restore" 297 | pod.ObjectMeta.Annotations["snapshotPath"] = checkpointPath 298 | if err := r.Create(ctx, pod); err != nil { 299 | return nil, err 300 | } 301 | return pod, nil 302 | } 303 | 304 | func (r *PodmigrationReconciler) removeCheckpointPod(ctx context.Context, pod *corev1.Pod, snapshotPathCurrent, newPodName, namespace string) error { 305 | if newPodName != "" { 306 | for { 307 | ok, _ := r.checkPodExist(ctx, newPodName, namespace) 308 | if ok != nil { 309 | break 310 | } 311 | } 312 | } 313 | snapshotPolicyUpdate := "" 314 | snapshotPathUpdate := "" 315 | if err := r.updateAnnotations(ctx, pod, snapshotPolicyUpdate, snapshotPathUpdate); err != nil { 316 | return err 317 | } 318 | os.Chmod(snapshotPathCurrent, 0777) 319 | if _, err := exec.Command("sudo", "rm", "-rf", snapshotPathCurrent).Output(); err != nil { 320 | return err 321 | } 322 | return nil 323 | } 324 | 325 | func (r *PodmigrationReconciler) updateAnnotations(ctx context.Context, pod *corev1.Pod, snapshotPolicy, snapshotPath string) error { 326 | ann := pod.ObjectMeta.Annotations 327 | if ann == nil { 328 | ann = make(map[string]string) 329 | } 330 | ann["snapshotPolicy"] = snapshotPolicy 331 | ann["snapshotPath"] = snapshotPath 332 | pod.ObjectMeta.Annotations = ann 333 | if err := r.Update(ctx, pod); err != nil { 334 | return err 335 | } 336 | return nil 337 | } 338 | 339 | func (r *PodmigrationReconciler) checkPodExist(ctx context.Context, name, namespace string) (*corev1.Pod, error) { 340 | var childPods corev1.PodList 341 | if err := r.List(ctx, &childPods, client.InNamespace(namespace)); err != nil { 342 | return nil, err 343 | } 344 | if len(childPods.Items) > 0 { 345 | for _, pod := range childPods.Items { 346 | if pod.Name == name && pod.Status.Phase == "Running" { 347 | return &pod, nil 348 | } 349 | } 350 | 351 | } 352 | return nil, nil 353 | } 354 | 355 | func (r *PodmigrationReconciler) getSourcePodTemplate(ctx context.Context, sourcePodName string, namespace string) (*corev1.PodTemplateSpec, error) { 356 | sourcePod, err := r.checkPodExist(ctx, sourcePodName, namespace) 357 | if sourcePod == nil { 358 | return nil, err 359 | } 360 | //(TODO: TuongVX): Get template of pod with multiple containers 361 | pod := sourcePod.DeepCopy() 362 | container := pod.Spec.Containers[0] 363 | template := &corev1.PodTemplateSpec{ 364 | ObjectMeta: pod.ObjectMeta, 365 | Spec: corev1.PodSpec{ 366 | Containers: []corev1.Container{ 367 | { 368 | Name: container.Name, 369 | Image: container.Image, 370 | Ports: container.Ports, 371 | VolumeMounts: container.VolumeMounts, 372 | }, 373 | }, 374 | Volumes: pod.Spec.Volumes, 375 | }, 376 | } 377 | return template, nil 378 | } 379 | 380 | func (r *PodmigrationReconciler) SetupWithManager(mgr ctrl.Manager) error { 381 | ctx := context.Background() 382 | if err := mgr.GetFieldIndexer().IndexField(ctx, &corev1.Pod{}, podOwnerKey, func(raw runtime.Object) []string { 383 | pod := raw.(*corev1.Pod) 384 | owner := metav1.GetControllerOf(pod) 385 | if owner == nil { 386 | return nil 387 | } 388 | if owner.Kind != "Podmigration" { 389 | return nil 390 | } 391 | 392 | return []string{owner.Name} 393 | }); err != nil { 394 | return err 395 | } 396 | return ctrl.NewControllerManagedBy(mgr). 397 | For(&podmigv1.Podmigration{}). 398 | Complete(r) 399 | } 400 | -------------------------------------------------------------------------------- /controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "path/filepath" 21 | "testing" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/rest" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/envtest" 29 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 30 | logf "sigs.k8s.io/controller-runtime/pkg/log" 31 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 32 | 33 | podmigv1 "github.com/SSU-DCN/podmigration-operator/api/v1" 34 | // +kubebuilder:scaffold:imports 35 | ) 36 | 37 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 38 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 39 | 40 | var cfg *rest.Config 41 | var k8sClient client.Client 42 | var testEnv *envtest.Environment 43 | 44 | func TestAPIs(t *testing.T) { 45 | RegisterFailHandler(Fail) 46 | 47 | RunSpecsWithDefaultAndCustomReporters(t, 48 | "Controller Suite", 49 | []Reporter{printer.NewlineReporter{}}) 50 | } 51 | 52 | var _ = BeforeSuite(func(done Done) { 53 | logf.SetLogger(zap.LoggerTo(GinkgoWriter, true)) 54 | 55 | By("bootstrapping test environment") 56 | testEnv = &envtest.Environment{ 57 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 58 | } 59 | 60 | var err error 61 | cfg, err = testEnv.Start() 62 | Expect(err).ToNot(HaveOccurred()) 63 | Expect(cfg).ToNot(BeNil()) 64 | 65 | err = podmigv1.AddToScheme(scheme.Scheme) 66 | Expect(err).NotTo(HaveOccurred()) 67 | 68 | // +kubebuilder:scaffold:scheme 69 | 70 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 71 | Expect(err).ToNot(HaveOccurred()) 72 | Expect(k8sClient).ToNot(BeNil()) 73 | 74 | close(done) 75 | }, 60) 76 | 77 | var _ = AfterSuite(func() { 78 | By("tearing down the test environment") 79 | err := testEnv.Stop() 80 | Expect(err).ToNot(HaveOccurred()) 81 | }) 82 | -------------------------------------------------------------------------------- /criu-3.14.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/criu-3.14.tar.bz2 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/SSU-DCN/podmigration-operator 2 | 3 | go 1.15 4 | 5 | // require ( 6 | // github.com/emicklei/go-restful v2.9.5+incompatible 7 | // github.com/go-logr/logr v0.1.0 8 | // github.com/onsi/ginkgo v1.11.0 9 | // github.com/onsi/gomega v1.8.1 10 | // k8s.io/api v0.17.2 11 | // k8s.io/apimachinery v0.17.2 12 | // k8s.io/client-go v0.17.2 13 | // sigs.k8s.io/controller-runtime v0.5.0 14 | // ) 15 | require ( 16 | github.com/emicklei/go-restful v2.9.5+incompatible 17 | github.com/go-logr/logr v0.1.0 18 | github.com/onsi/ginkgo v1.12.1 19 | github.com/onsi/gomega v1.10.1 20 | github.com/spf13/cobra v0.0.5 21 | gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e // indirect 22 | k8s.io/api v0.18.6 23 | k8s.io/apimachinery v0.18.6 24 | k8s.io/client-go v0.18.6 25 | sigs.k8s.io/controller-runtime v0.6.3 26 | sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06 // indirect 27 | ) 28 | 29 | replace k8s.io/api => ../kubernetes/staging/src/k8s.io/api 30 | 31 | replace k8s.io/apimachinery => ../kubernetes/staging/src/k8s.io/apimachinery 32 | 33 | replace k8s.io/client-go => ../kubernetes/staging/src/k8s.io/client-go 34 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= 4 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 5 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 6 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 7 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 8 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 9 | cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= 10 | cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= 11 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 12 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 13 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 14 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 15 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 16 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= 17 | github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= 18 | github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= 19 | github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= 20 | github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= 21 | github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= 22 | github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= 23 | github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 24 | github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 25 | github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= 26 | github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= 27 | github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= 28 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 29 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 30 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 31 | github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 32 | github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 33 | github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 34 | github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 35 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 36 | github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= 37 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 38 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 39 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= 40 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 41 | github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 42 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 43 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 44 | github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= 45 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 46 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 47 | github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= 48 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 49 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 50 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 51 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 52 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 53 | github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= 54 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 55 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 56 | github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= 57 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 58 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 59 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 60 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 61 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 62 | github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 63 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 64 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 65 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 66 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 67 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 68 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 69 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 70 | github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 71 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 72 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 73 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= 74 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= 75 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= 76 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 77 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 78 | github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 79 | github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 80 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 81 | github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= 82 | github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 83 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 84 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 85 | github.com/evanphx/json-patch v0.0.0-20190815234213-e83c0a1c26c8/go.mod h1:pmLOTb3x90VhIKxsA9yeQG5yfOkkKnkk1h+Ql8NDYDw= 86 | github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 87 | github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= 88 | github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 89 | github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= 90 | github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 91 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 92 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 93 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 94 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 95 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 96 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 97 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 98 | github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 99 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 100 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 101 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 102 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 103 | github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= 104 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 105 | github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= 106 | github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= 107 | github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= 108 | github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 109 | github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 110 | github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= 111 | github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= 112 | github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 113 | github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 114 | github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 115 | github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= 116 | github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 117 | github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 118 | github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= 119 | github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 120 | github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= 121 | github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 122 | github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 123 | github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= 124 | github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= 125 | github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 126 | github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 127 | github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 128 | github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= 129 | github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= 130 | github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= 131 | github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= 132 | github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= 133 | github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= 134 | github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 135 | github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 136 | github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= 137 | github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= 138 | github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 139 | github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 140 | github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= 141 | github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= 142 | github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= 143 | github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 144 | github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 145 | github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 146 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 147 | github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= 148 | github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= 149 | github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= 150 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 151 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 152 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 153 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= 154 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 155 | github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= 156 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 157 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 158 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 159 | github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE= 160 | github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 161 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 162 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 163 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= 164 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 165 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 166 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 167 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 168 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 169 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 170 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 171 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 172 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 173 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 174 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 175 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 176 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 177 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 178 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 179 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 180 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 181 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 182 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 183 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 184 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 185 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 186 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 187 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 188 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 189 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 190 | github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= 191 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 192 | github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= 193 | github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 194 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 195 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 196 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 197 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 198 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 199 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 200 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 201 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 202 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 203 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 204 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 205 | github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 206 | github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= 207 | github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= 208 | github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= 209 | github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= 210 | github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= 211 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 212 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 213 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 214 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 215 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 216 | github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 217 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 218 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 219 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 220 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 221 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 222 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 223 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 224 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 225 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 226 | github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 227 | github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= 228 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 229 | github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= 230 | github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 231 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 232 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 233 | github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 234 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 235 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 236 | github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= 237 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 238 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 239 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 240 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 241 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 242 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 243 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 244 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 245 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 246 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 247 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 248 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 249 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 250 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 251 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 252 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 253 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 254 | github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= 255 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 256 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 257 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 258 | github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 259 | github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 260 | github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 261 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 262 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 263 | github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 264 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 265 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 266 | github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 267 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 268 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 269 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 270 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 271 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 272 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 273 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 274 | github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 275 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 276 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 277 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 278 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 279 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 280 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 281 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 282 | github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= 283 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 284 | github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= 285 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 286 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 287 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 288 | github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= 289 | github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 290 | github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= 291 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 292 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 293 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 294 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 295 | github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= 296 | github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= 297 | github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= 298 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 299 | github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 300 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 301 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 302 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 303 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 304 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 305 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 306 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 307 | github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 308 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 309 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 310 | github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= 311 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 312 | github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= 313 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 314 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 315 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= 316 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 317 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= 318 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 319 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 320 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 321 | github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= 322 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 323 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 324 | github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= 325 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 326 | github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= 327 | github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 328 | github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= 329 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 330 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 331 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 332 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 333 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 334 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 335 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 336 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 337 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 338 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 339 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 340 | github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= 341 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 342 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 343 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 344 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 345 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 346 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 347 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 348 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 349 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 350 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 351 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 352 | github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 353 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 354 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 355 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 356 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 357 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 358 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 359 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 360 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 361 | github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= 362 | github.com/vutuong/kubernetes v1.18.3 h1:vk47EmKAWhKs+V+Ei9ya2e6Vud3Sh5dMc8b+oC895o8= 363 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 364 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 365 | go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 366 | go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= 367 | go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 368 | go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 369 | go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 370 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 371 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 372 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 373 | go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= 374 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 375 | go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= 376 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 377 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 378 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 379 | go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= 380 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 381 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 382 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 383 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 384 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 385 | golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 386 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 387 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 388 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 389 | golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 390 | golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= 391 | golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 392 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 393 | golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 394 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= 395 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 396 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 397 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 398 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 399 | golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 400 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 401 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 402 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 403 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 404 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 405 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 406 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 407 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 408 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 409 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 410 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 411 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 412 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 413 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 414 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 415 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 416 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 417 | golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 418 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 419 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 420 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 421 | golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 422 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 423 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 424 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 425 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 426 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 427 | golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 428 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 429 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 430 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 431 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 432 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 433 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 434 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 435 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 436 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= 437 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 438 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 439 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= 440 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 441 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= 442 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 443 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 444 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 445 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 446 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 447 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= 448 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 449 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 450 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 451 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 452 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 453 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 454 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 455 | golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 456 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 457 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 458 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 459 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 460 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 461 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 462 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 463 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 464 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 465 | golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 466 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 467 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 468 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 469 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 470 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 471 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 472 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 473 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= 474 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 475 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 476 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 477 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 478 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 479 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 480 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 481 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 482 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= 483 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 484 | golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 485 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 486 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 487 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 488 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 489 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 490 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 491 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 492 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 493 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= 494 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 495 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= 496 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 497 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 498 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 499 | golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 500 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 501 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 502 | golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 503 | golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 504 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 505 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 506 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 507 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 508 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 509 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 510 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 511 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 512 | golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 513 | golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 514 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 515 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 516 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 517 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 518 | golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 519 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 520 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 521 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 522 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= 523 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 524 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 525 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 526 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 527 | gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= 528 | gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= 529 | gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= 530 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 531 | gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= 532 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 533 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 534 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 535 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 536 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 537 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 538 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 539 | google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= 540 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 541 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 542 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 543 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 544 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 545 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 546 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 547 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 548 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 549 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 550 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 551 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 552 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 553 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 554 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 555 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 556 | google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 557 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 558 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 559 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 560 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 561 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 562 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 563 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 564 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 565 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 566 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 567 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 568 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 569 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 570 | gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= 571 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 572 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 573 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 574 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 575 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 576 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 577 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 578 | gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= 579 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 580 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 581 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 582 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 583 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 584 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 585 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 586 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 587 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 588 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 589 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 590 | gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 591 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 592 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 593 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 594 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 595 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 596 | k8s.io/api v0.17.2 h1:NF1UFXcKN7/OOv1uxdRz3qfra8AHsPav5M93hlV9+Dc= 597 | k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= 598 | k8s.io/apiextensions-apiserver v0.17.2 h1:cP579D2hSZNuO/rZj9XFRzwJNYb41DbNANJb6Kolpss= 599 | k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= 600 | k8s.io/apiextensions-apiserver v0.18.6 h1:vDlk7cyFsDyfwn2rNAO2DbmUbvXy5yT5GE3rrqOzaMo= 601 | k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= 602 | k8s.io/apimachinery v0.17.2 h1:hwDQQFbdRlpnnsR64Asdi55GyCaIP/3WQpMmbNBeWr4= 603 | k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= 604 | k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= 605 | k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= 606 | k8s.io/client-go v0.17.2 h1:ndIfkfXEGrNhLIgkr0+qhRguSD3u6DCmonepn1O6NYc= 607 | k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI= 608 | k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= 609 | k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= 610 | k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= 611 | k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= 612 | k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 613 | k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 614 | k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 615 | k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 616 | k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 617 | k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 618 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 619 | k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 620 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 621 | k8s.io/klog/v2 v2.1.0 h1:X3+Mru/L3jy4BI4vcAYkHvL6PyU+QBsuhEqwlI4mgkA= 622 | k8s.io/klog/v2 v2.1.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 623 | k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= 624 | k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= 625 | k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= 626 | k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9 h1:5NC2ITmvg8RoxoH0wgmL4zn4VZqXGsKbxrikjaQx6s4= 627 | k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw= 628 | k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= 629 | k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= 630 | k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= 631 | k8s.io/utils v0.0.0-20200414100711-2df71ebbae66 h1:Ly1Oxdu5p5ZFmiVT71LFgeZETvMfZ1iBIGeOenT2JeM= 632 | k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 633 | k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 h1:v8ud2Up6QK1lNOKFgiIVrZdMg7MpmSnvtrOieolJKoE= 634 | k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 635 | modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= 636 | modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= 637 | modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= 638 | modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= 639 | modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= 640 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 641 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= 642 | sigs.k8s.io/controller-runtime v0.5.0 h1:CbqIy5fbUX+4E9bpnBFd204YAzRYlM9SWW77BbrcDQo= 643 | sigs.k8s.io/controller-runtime v0.5.0/go.mod h1:REiJzC7Y00U+2YkMbT8wxgrsX5USpXKGhb2sCtAXiT8= 644 | sigs.k8s.io/controller-runtime v0.6.3 h1:SBbr+inLPEKhvlJtrvDcwIpm+uhDvp63Bl72xYJtoOE= 645 | sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= 646 | sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= 647 | sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06 h1:zD2IemQ4LmOcAumeiyDWXKUI2SO0NYDe3H6QGvPOVgU= 648 | sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= 649 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= 650 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= 651 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= 652 | sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= 653 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 654 | sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= 655 | sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= 656 | -------------------------------------------------------------------------------- /gui/.browserslistrc: -------------------------------------------------------------------------------- 1 | > 1% 2 | last 2 versions 3 | not dead 4 | -------------------------------------------------------------------------------- /gui/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true 5 | }, 6 | extends: ["plugin:vue/essential", "eslint:recommended", "@vue/prettier"], 7 | parserOptions: { 8 | parser: "babel-eslint" 9 | }, 10 | rules: { 11 | "no-console": process.env.NODE_ENV === "production" ? "warn" : "off", 12 | "no-debugger": process.env.NODE_ENV === "production" ? "warn" : "off" 13 | }, 14 | overrides: [ 15 | { 16 | files: [ 17 | "**/__tests__/*.{j,t}s?(x)", 18 | "**/tests/unit/**/*.spec.{j,t}s?(x)" 19 | ], 20 | env: { 21 | jest: true 22 | } 23 | } 24 | ] 25 | }; 26 | -------------------------------------------------------------------------------- /gui/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | /tests/e2e/videos/ 6 | /tests/e2e/screenshots/ 7 | 8 | # local env files 9 | .env.local 10 | .env.*.local 11 | 12 | # Log files 13 | npm-debug.log* 14 | yarn-debug.log* 15 | yarn-error.log* 16 | 17 | # Editor directories and files 18 | .idea 19 | .vscode 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /gui/README.md: -------------------------------------------------------------------------------- 1 | # Podmigration-gui 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | ### Note: You might need to install specific version: 8 | sudo npm install -g npm@2.1.7 9 | ``` 10 | 11 | ### Compiles and hot-reloads for development 12 | ``` 13 | npm run serve 14 | ``` 15 | 16 | ### Compiles and minifies for production 17 | ``` 18 | npm run build 19 | ``` 20 | 21 | ### Run your unit tests 22 | ``` 23 | npm run test:unit 24 | ``` 25 | 26 | ### Run your end-to-end tests 27 | ``` 28 | npm run test:e2e 29 | ``` 30 | 31 | ### Lints and fixes files 32 | ``` 33 | npm run lint 34 | ``` 35 | 36 | ### Customize configuration 37 | See [Configuration Reference](https://cli.vuejs.org/config/). 38 | -------------------------------------------------------------------------------- /gui/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: ["@vue/cli-plugin-babel/preset"] 3 | }; 4 | -------------------------------------------------------------------------------- /gui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "darkroom-gui", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve --port 3000", 7 | "build": "vue-cli-service build", 8 | "lint": "vue-cli-service lint" 9 | }, 10 | "dependencies": { 11 | "@mdi/font": "^3.6.95", 12 | "axios": "^0.21.1", 13 | "core-js": "^3.6.4", 14 | "roboto-fontface": "*", 15 | "vue": "^2.6.11", 16 | "vue-router": "^3.1.6", 17 | "vuetify": "^2.2.27", 18 | "vuex": "^3.1.3" 19 | }, 20 | "devDependencies": { 21 | "@vue/cli-plugin-babel": "~4.3.0", 22 | "@vue/cli-plugin-eslint": "~4.3.0", 23 | "@vue/cli-plugin-router": "~4.3.0", 24 | "@vue/cli-plugin-vuex": "~4.3.0", 25 | "@vue/cli-service": "^4.3.1", 26 | "@vue/eslint-config-prettier": "^6.0.0", 27 | "axios-mock-adapter": "^1.18.1", 28 | "babel-eslint": "^10.1.0", 29 | "eslint": "^6.7.2", 30 | "eslint-plugin-prettier": "^3.1.1", 31 | "eslint-plugin-vue": "^6.2.2", 32 | "lint-staged": "^9.5.0", 33 | "lodash": "^4.17.15", 34 | "node-sass": "^4.12.0", 35 | "prettier": "^1.19.1", 36 | "sass": "^1.19.0", 37 | "sass-loader": "^8.0.2", 38 | "vue-cli-plugin-pug": "~1.0.7", 39 | "vue-cli-plugin-vuetify": "~2.0.5", 40 | "vue-template-compiler": "^2.6.11", 41 | "vuetify-loader": "^1.3.0" 42 | }, 43 | "gitHooks": { 44 | "pre-commit": "lint-staged" 45 | }, 46 | "lint-staged": { 47 | "*.{js,jsx,vue}": [ 48 | "vue-cli-service lint", 49 | "git add" 50 | ] 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /gui/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/gui/public/favicon.ico -------------------------------------------------------------------------------- /gui/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | <%= htmlWebpackPlugin.options.title %> 9 | 10 | 11 | 14 |
15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /gui/src/App.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 17 | -------------------------------------------------------------------------------- /gui/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/gui/src/assets/logo.png -------------------------------------------------------------------------------- /gui/src/assets/logo.svg: -------------------------------------------------------------------------------- 1 | Artboard 46 2 | -------------------------------------------------------------------------------- /gui/src/components/PodmigrationCard.vue: -------------------------------------------------------------------------------- 1 | 23 | 24 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /gui/src/components/SidebarHeader.vue: -------------------------------------------------------------------------------- 1 | 29 | 30 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /gui/src/components/utils/iconClassMap.js: -------------------------------------------------------------------------------- 1 | export default new Map([ 2 | ["checkpoint", "mdi-folder-multiple-image"], 3 | ["restore", "mdi-amazon"], 4 | ["live-migration", "mdi-google"] 5 | ]); 6 | -------------------------------------------------------------------------------- /gui/src/components/utils/statusColorMap.js: -------------------------------------------------------------------------------- 1 | export default new Map([ 2 | ["disabled", "disabled"], 3 | ["deploying", "warning"], 4 | ["deployed", "success"], 5 | ["failed", "error"] 6 | ]); 7 | -------------------------------------------------------------------------------- /gui/src/filters.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | 3 | Vue.filter("capitalize", function(value) { 4 | if (!value) return ""; 5 | return ( 6 | value 7 | .toString() 8 | .charAt(0) 9 | .toUpperCase() + value.slice(1) 10 | ); 11 | }); 12 | 13 | Vue.filter("titleCase", function(value) { 14 | if (!value) return ""; 15 | return value.toString().replace(/([^\W_]+[^\s-]*) */g, function(s) { 16 | return s.charAt(0).toUpperCase() + s.substr(1).toLowerCase(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /gui/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import App from "./App.vue"; 3 | import router from "./router"; 4 | import store from "./store"; 5 | import "./mixins"; 6 | import "./filters"; 7 | import vuetify from "./plugins/vuetify"; 8 | import "roboto-fontface/css/roboto/roboto-fontface.css"; 9 | import "@mdi/font/css/materialdesignicons.css"; 10 | import Podmigration from "./services/podmigration"; 11 | 12 | Vue.config.productionTip = false; 13 | 14 | function VUE_APP() { 15 | // setup the HTTP API namespace 16 | Vue.prototype.$api = new Podmigration(); 17 | 18 | new Vue({ 19 | router, 20 | store, 21 | vuetify, 22 | render: h => h(App) 23 | }).$mount("#app"); 24 | } 25 | 26 | function SETUP() { 27 | VUE_APP(); 28 | } 29 | 30 | SETUP(); 31 | -------------------------------------------------------------------------------- /gui/src/mixins.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | 3 | function getTitle(vm) { 4 | const { title } = vm.$options; 5 | if (title) { 6 | return typeof title === "function" ? title.call(vm) : title; 7 | } 8 | } 9 | 10 | Vue.mixin({ 11 | created() { 12 | const title = getTitle(this); 13 | if (title) { 14 | document.title = `${title} | Podmigration`; 15 | } 16 | } 17 | }); 18 | -------------------------------------------------------------------------------- /gui/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import Vuetify from "vuetify/lib"; 3 | 4 | Vue.use(Vuetify); 5 | 6 | export default new Vuetify({ 7 | theme: { 8 | options: { 9 | customProperties: true 10 | }, 11 | themes: { 12 | light: { 13 | primary: "#31b057", 14 | secondary: "#388bf2", 15 | accent: "#dbebff", 16 | error: "#FF5252", 17 | info: "#2196F3", 18 | success: "#4CAF50", 19 | warning: "#FFC107", 20 | disabled: "#a0a4a8" 21 | } 22 | } 23 | } 24 | }); 25 | -------------------------------------------------------------------------------- /gui/src/router/index.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import VueRouter from "vue-router"; 3 | import SourceList from "../views/SourceList"; 4 | import SourceDetail from "../views/SourceDetail"; 5 | import Dashboard from "../views/Dashboard"; 6 | import AddSource from "../views/AddSource"; 7 | 8 | Vue.use(VueRouter); 9 | 10 | const routes = [ 11 | { 12 | path: "/", 13 | name: "dashboard", 14 | component: Dashboard, 15 | meta: { requiresAuth: true } 16 | }, 17 | { 18 | path: "/sources", 19 | name: "sources", 20 | component: SourceList, 21 | meta: { requiresAuth: true } 22 | }, 23 | { 24 | path: "/sources/new", 25 | name: "add-source", 26 | component: AddSource, 27 | meta: { requiresAuth: true } 28 | }, 29 | { 30 | path: "/sources/:name", 31 | name: "source-detail", 32 | component: SourceDetail, 33 | meta: { requiresAuth: true } 34 | }, 35 | { 36 | path: "/support", 37 | name: "support", 38 | component: () => import("../views/Support") 39 | } 40 | ]; 41 | 42 | const router = new VueRouter({ 43 | mode: "history", 44 | base: process.env.BASE_URL, 45 | routes 46 | }); 47 | 48 | export default router; 49 | -------------------------------------------------------------------------------- /gui/src/services/mock.js: -------------------------------------------------------------------------------- 1 | import MockAdapter from "axios-mock-adapter"; 2 | 3 | export default class Mock { 4 | constructor(axios) { 5 | const mockDelay = 0; 6 | 7 | this.mock = new MockAdapter(axios, { delayResponse: mockDelay }); 8 | this.mock.injectMocks = () => { 9 | return this.mock; 10 | }; 11 | } 12 | 13 | setupPluginMocks() { 14 | this.mock 15 | .injectMocks() // additional mocks added from RestClient 16 | .onAny() 17 | .passThrough(); 18 | } 19 | 20 | setupMockEndpoints() { 21 | console.warn( 22 | "%c ✨You are mocking api requests.", 23 | "background: gray; color: white; display: block; padding: 0.25rem;" 24 | ); 25 | 26 | this.mock 27 | // .onGet("/api/podmigrations") 28 | // .reply(200, { 29 | // items: [ 30 | // { 31 | // name: "podmigration-sample", 32 | // version: "0.1.0", 33 | // source: { 34 | // type: "WebFolder" 35 | // }, 36 | // domains: [ 37 | // "students.podmigration.gojek.io" 38 | // ] 39 | // } 40 | // ] 41 | // }) 42 | .onAny() 43 | .passThrough(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /gui/src/services/podmigration.js: -------------------------------------------------------------------------------- 1 | import RestClient from "./restClient"; 2 | 3 | // export default class Podmigration { 4 | export default class Podmigration { 5 | constructor(options) { 6 | const opts = options || {}; 7 | 8 | this.options = opts; 9 | this.client = new RestClient(opts); 10 | } 11 | 12 | buildUrl(path) { 13 | return this.client.buildUrl(path); 14 | } 15 | 16 | /** 17 | * Info 18 | */ 19 | getInfo() { 20 | return this.client.get("/"); 21 | } 22 | 23 | getStatus() { 24 | return this.client.status("/"); 25 | } 26 | 27 | /** 28 | * Podmigrations 29 | */ 30 | // get a list of all Podmigrations 31 | getAllPodmigrations(params) { 32 | return this.client.get("/api/Podmigrations", params); 33 | } 34 | createPodmigration(params) { 35 | console.log(params); 36 | return this.client.post("/api/Podmigrations", params); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /gui/src/services/restClient.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import Mock from "./mock"; 3 | 4 | export default class RestClient { 5 | constructor(options) { 6 | const opts = options || {}; 7 | 8 | // this.host = opts.url 9 | 10 | // leave this blank! 11 | this.headers = {}; 12 | 13 | // RestClient.setupMocks(opts.injectMocks) 14 | 15 | /** 16 | * We no longer need to run this because the setup is done 17 | * at app launch before anything else happens. 18 | */ 19 | // RestClient.apiConfig() 20 | 21 | RestClient.setupMocks(opts.injectMocks); 22 | this.client = RestClient.axiosInit(); 23 | } 24 | 25 | /** 26 | * axiosInit 27 | * 28 | * This function creates the Axios endpoint with the 29 | * value from localStorage 30 | */ 31 | static axiosInit() { 32 | return axios.create({ 33 | headers: this.headers, 34 | ...this.axiosConfig 35 | }); 36 | } 37 | 38 | /** 39 | * Setup mock endpoints that override axios calls 40 | * @param {Object} injectMocks - the mock endpoint functions defined by any 41 | * external plugins - if they exist. If not, then it passes the real (unmocked) response. 42 | */ 43 | static setupMocks(injectMocks) { 44 | const mock = new Mock(axios); 45 | if (process.env.VUE_APP_MOCK_API_ENABLED === "true") { 46 | mock.setupMockEndpoints(); 47 | } else { 48 | if (injectMocks && injectMocks.length) { 49 | injectMocks.forEach(injectedMock => { 50 | injectedMock(mock.mock); 51 | }); 52 | } 53 | 54 | mock.setupPluginMocks(); 55 | } 56 | } 57 | 58 | buildUrl(path) { 59 | return `${this.host}${path}`; 60 | } 61 | 62 | async status(path, options) { 63 | const opts = (await options) || {}; 64 | const url = await path; 65 | const client = await this.client; 66 | 67 | return client 68 | .get(url, opts) 69 | .then(response => { 70 | return response.statusText; 71 | }) 72 | .catch(error => { 73 | console.error(error); 74 | 75 | return error; 76 | }); 77 | } 78 | 79 | async get(path, options) { 80 | const opts = (await options) || {}; 81 | // const url = this.buildUrl(path) 82 | const url = await path; 83 | const client = await this.client; 84 | 85 | return client 86 | .get(url, opts) 87 | .then(response => { 88 | return response.data; 89 | }) 90 | .catch(error => { 91 | console.error(error); 92 | }); 93 | } 94 | 95 | async post(path, options) { 96 | const opts = (await options) || {}; 97 | // const url = this.buildUrl(path) 98 | const url = await path; 99 | const client = await this.client; 100 | 101 | return client 102 | .post(url, opts) 103 | .then(response => { 104 | return response.data; 105 | }) 106 | .catch(error => { 107 | console.error(error); 108 | throw error; 109 | }); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /gui/src/store/index.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import Vuex from "vuex"; 3 | import modules from "./modules"; 4 | 5 | Vue.use(Vuex); 6 | 7 | const debug = process.env.NODE_ENV !== "production"; 8 | 9 | export default new Vuex.Store({ 10 | strict: debug, 11 | modules 12 | }); 13 | -------------------------------------------------------------------------------- /gui/src/store/modules/app.js: -------------------------------------------------------------------------------- 1 | const state = { 2 | items: [ 3 | { 4 | icon: "mdi-desktop-mac-dashboard", 5 | text: "Dashboard", 6 | requiresAuth: true, 7 | path: "/" 8 | }, 9 | { 10 | icon: "mdi-image-search", 11 | text: "Sources", 12 | search: true, 13 | requiresAuth: true, 14 | path: "/sources" 15 | }, 16 | { icon: "mdi-lifebuoy", text: "Support", path: "/support" } 17 | ] 18 | }; 19 | 20 | const getters = { 21 | items(state) { 22 | return state.items; 23 | // .filter( 24 | // e => 25 | // e.requiresAuth === rootGetters["auth/isLoggedIn"] || 26 | // e.requiresAuth == null 27 | // ); 28 | } 29 | }; 30 | 31 | const mutations = {}; 32 | 33 | const actions = {}; 34 | 35 | export { state, mutations, actions, getters }; 36 | -------------------------------------------------------------------------------- /gui/src/store/modules/auth.js: -------------------------------------------------------------------------------- 1 | const state = { 2 | isLoggedIn: false, 3 | thumbnailUrl: null 4 | }; 5 | 6 | const getters = { 7 | isLoggedIn: state => state.isLoggedIn, 8 | thumbnailUrl: state => state.thumbnailUrl 9 | }; 10 | 11 | const mutations = { 12 | LOGIN(state) { 13 | state.isLoggedIn = true; 14 | }, 15 | LOGOUT(state) { 16 | state.isLoggedIn = false; 17 | }, 18 | SET_THUMBNAIL_URL(state, URL) { 19 | state.thumbnailUrl = URL; 20 | } 21 | }; 22 | 23 | const actions = { 24 | login({ commit }) { 25 | commit("LOGIN"); 26 | }, 27 | logout({ commit }) { 28 | commit("LOGOUT"); 29 | } 30 | }; 31 | 32 | export { state, mutations, actions, getters }; 33 | -------------------------------------------------------------------------------- /gui/src/store/modules/index.js: -------------------------------------------------------------------------------- 1 | import camelCase from "lodash/camelCase"; 2 | 3 | const requireModule = require.context(".", false, /\.js$/); 4 | const modules = {}; 5 | 6 | requireModule.keys().forEach(fileName => { 7 | // Don't register this file as a Vuex module 8 | if (fileName === "./index.js") return; 9 | 10 | const moduleName = camelCase(fileName.replace(/(\.\/|\.js)/g, "")); 11 | modules[moduleName] = { 12 | namespaced: true, 13 | ...requireModule(fileName) 14 | }; 15 | }); 16 | 17 | export default modules; 18 | -------------------------------------------------------------------------------- /gui/src/views/AddSource.vue: -------------------------------------------------------------------------------- 1 | 120 | 121 | 187 | 188 | 189 | -------------------------------------------------------------------------------- /gui/src/views/Dashboard.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /gui/src/views/SourceDetail.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /gui/src/views/SourceList.vue: -------------------------------------------------------------------------------- 1 | 21 | 22 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /gui/src/views/Support.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /gui/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | transpileDependencies: ["vuetify"], 3 | devServer: { 4 | proxy: { 5 | "^/api": { 6 | target: "http://localhost:5000", 7 | secure: false, 8 | pathRewrite: { "^/api": "" }, 9 | changeOrigin: true, 10 | logLevel: "debug" 11 | } 12 | } 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /init-cluster-containerd-CRIU.md: -------------------------------------------------------------------------------- 1 | ## This document guides how to bootstrap Kubernetes cluster without Docker and enable CRIU intergration for Podmigration feature. 2 | ### Step0: 3 | - Ubuntu 18.04 4 | - Do step 1-6 at all nodes. 5 | - Do step 7 at controller node. 6 | - Do step 8 at each worker nodes 7 | ### Step1: Install container runtime - Containerd 8 | - Download containerd and unpackage: 9 | ``` 10 | $ sudo apt-get update 11 | $ sudo apt-get install gcc 12 | 13 | $ mkdir tmp 14 | $ cd tmp/ 15 | $ sudo wget https://golang.org/dl/go1.15.5.linux-amd64.tar.gz 16 | $ sudo tar -xzf go1.15.5.linux-amd64.tar.gz 17 | $ sudo mv go /usr/local 18 | $ sudo vi $HOME/.profile 19 | export GOROOT=/usr/local/go 20 | export GOPATH=$HOME/go 21 | export GOBIN=$GOPATH/bin 22 | export PATH=$GOROOT/bin:$GOBIN:$PATH 23 | $ source $HOME/.profile 24 | $ go version 25 | 26 | $ sudo apt install make 27 | $ wget https://github.com/containerd/containerd/releases/download/v1.3.6/containerd-1.3.6-linux-amd64.tar.gz 28 | $ mkdir containerd 29 | $ tar -xvf containerd-1.3.6-linux-amd64.tar.gz -C containerd 30 | $ sudo mv containerd/bin/* /bin/ 31 | ``` 32 | - Replace the containerd-cri with interface extentions supporting CRIU, which is need for running podmigration. There are two options: 33 | 1. Build from source: 34 | ``` 35 | $ git clone https://github.com/vutuong/containerd-cri.git 36 | $ cd containerd-cri/ 37 | $ go version 38 | $ go get github.com/containerd/cri/cmd/containerd 39 | $ make 40 | $ sudo make install 41 | $ cd _output/ 42 | $ sudo mv containerd /bin/ 43 | ``` 44 | 2. Download binaries: 45 | ``` 46 | $ cd containerd/ 47 | $ wget https://k8s-pod-migration.obs.eu-de.otc.t-systems.com/v2/containerd 48 | $ git clone https://github.com/SSU-DCN/podmigration-operator.git 49 | $ cd podmigration-operator 50 | $ tar -vxf binaries.tar.bz2 51 | $ cd custom-binaries/ 52 | $ chmod +x containerd 53 | $ sudo mv containerd /bin/ 54 | ``` 55 | - Configure containerd and create the containerd configuration file 56 | ``` 57 | $ sudo mkdir /etc/containerd 58 | $ sudo nano /etc/containerd/config.toml 59 | 60 | [plugins] 61 | [plugins.cri.containerd] 62 | snapshotter = "overlayfs" 63 | [plugins.cri.containerd.default_runtime] 64 | runtime_type = "io.containerd.runtime.v1.linux" 65 | runtime_engine = "/usr/local/bin/runc" 66 | runtime_root = "" 67 | ``` 68 | - Install newest version of runc, at this time, I'm using v1.0.0-rc92 (the podmigration) 69 | ``` 70 | $ wget https://github.com/opencontainers/runc/releases/download/v1.0.0-rc92/runc.amd64 71 | $ whereis runc 72 | $ sudo mv runc.amd64 runc 73 | $ chmod +x runc 74 | $ sudo mv runc /usr/local/bin/ 75 | ``` 76 | - Configure containerd and create the containerd.service systemd unit file 77 | ``` 78 | $ sudo nano /etc/systemd/system/containerd.service 79 | 80 | [Unit] 81 | Description=containerd container runtime 82 | Documentation=https://containerd.io 83 | After=network.target 84 | 85 | [Service] 86 | ExecStartPre=/sbin/modprobe overlay 87 | ExecStart=/bin/containerd 88 | Restart=always 89 | RestartSec=5 90 | Delegate=yes 91 | KillMode=process 92 | OOMScoreAdjust=-999 93 | LimitNOFILE=1048576 94 | LimitNPROC=infinity 95 | LimitCORE=infinity 96 | 97 | [Install] 98 | WantedBy=multi-user.target 99 | ``` 100 | - Reload containerd service 101 | ``` 102 | $ sudo systemctl daemon-reload 103 | $ sudo systemctl restart containerd 104 | $ sudo systemctl status containerd 105 | ``` 106 | ### Step2: Solve a few problems introduced with containerd 107 | ``` 108 | $ sudo nano /etc/sysctl.conf 109 | ... 110 | net.bridge.bridge-nf-call-iptables = 1 111 | 112 | $ sudo -s 113 | $ sudo echo '1' > /proc/sys/net/ipv4/ip_forward 114 | $ exit 115 | $ sudo sysctl --system 116 | $ sudo modprobe overlay 117 | $ sudo modprobe br_netfilter 118 | ``` 119 | ### Step3: You'll need to map all of your nodes in ```/etc/hosts``` 120 | ### Step4: Install kubernetes components 121 | ``` 122 | $ curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add 123 | $ sudo apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main" 124 | $ sudo apt-get install kubeadm=1.19.0-00 kubelet=1.19.0-00 kubectl=1.19.0-00 -y 125 | $ whereis kubeadm 126 | $ whereis kubelet 127 | ``` 128 | 129 | ### Step5 : Replace kubelet with the custom kubelet. 130 | The kubelet and kubeadm binaries can be found in this folder. (If you already download this file at Step1, just go to directory custom-binaries/) 131 | ``` 132 | $ git clone https://github.com/vutuong/kubernetes.git 133 | $ git clone https://github.com/SSU-DCN/podmigration-operator.git 134 | $ cd podmigration-operator 135 | $ tar -vxf binaries.tar.bz2 136 | $ cd custom-binaries 137 | ``` 138 | - After downloading/building the kubelet and kubeadm binaries, replace it with existing default K8s-kubelet binaries: 139 | ``` 140 | $ chmod +x kubeadm kubelet 141 | $ sudo mv kubeadm kubelet /usr/bin/ 142 | $ sudo systemctl daemon-reload 143 | $ sudo systemctl restart kubelet 144 | $ sudo systemctl status kubelet 145 | ``` 146 | ### Step6: Init k8s-cluster. 147 | - In the master node run following command: 148 | ``` 149 | $ sudo kubeadm init --pod-network-cidr=10.244.0.0/16 150 | ``` 151 | - Use the output log command to join workernode 152 | - To run the kubectl commands: 153 | ``` 154 | $ kubectl get nodes 155 | $ mkdir -p $HOME/.kube 156 | $ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 157 | $ sudo chown $(id -u):$(id -g) $HOME/.kube/config 158 | ``` 159 | - Apply flannel CNI add-on network: 160 | ``` 161 | $ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml 162 | $ kubectl get pods -n kube-system 163 | ``` 164 | - Check custom kubelet is running: 165 | ``` 166 | $ journalctl -fu kubelet 167 | 168 | Feb 04 05:55:16 tuong-worker1 kubelet[26650]: I0204 05:55:16.979326 26650 kuberuntime_manager.go:841] Should we migrate?Runningfalse 169 | Feb 04 05:55:21 tuong-worker1 kubelet[26650]: I0204 05:55:21.979185 26650 kuberuntime_manager.go:841] Should we migrate?Runningfalse 170 | Feb 04 05:55:25 tuong-worker1 kubelet[26650]: I0204 05:55:25.979207 26650 kuberuntime_manager.go:841] Should we migrate?Runningfalse 171 | 172 | ``` 173 | ### Step7: You may need to change kubelet.config mode. (Or not) 174 | ``` 175 | $ sudo nano /var/lib/kubelet/config.yaml 176 | 177 | authorization: 178 | #mode: Webhook 179 | mode: AlwaysAllow 180 | ``` 181 | ### Step8: Install CRIU. 182 | ``` 183 | $ git clone https://github.com/SSU-DCN/podmigration-operator.git 184 | $ cd podmigration-operator 185 | $ tar -vxf criu-3.14.tar.bz2 186 | $ cd criu-3.14 187 | $ sudo apt-get install protobuf-c-compiler libprotobuf-c0-dev protobuf-compiler \ 188 | libprotobuf-dev:amd64 gcc build-essential bsdmainutils python git-core \ 189 | asciidoc make htop git curl supervisor cgroup-lite libapparmor-dev \ 190 | libseccomp-dev libprotobuf-dev libprotobuf-c0-dev protobuf-c-compiler \ 191 | protobuf-compiler python-protobuf libnl-3-dev libcap-dev libaio-dev \ 192 | apparmor libnet1-dev libnl-genl-3-dev libnl-route-3-dev libnfnetlink-dev pkg-config 193 | 194 | $ make clean 195 | $ make 196 | $ sudo make install 197 | $ criu check 198 | $ criu check --all 199 | 200 | $ mkdir /etc/criu 201 | $ touch /etc/criu/runc.conf 202 | $ nano /etc/criu/runc.conf 203 | tcp-established 204 | tcp-close 205 | ``` 206 | ### Step9: Config NFS shared folder for every node in the cluster. 207 | - Config NFS server at Master node 208 | ``` 209 | $ sudo apt-get update 210 | $ sudo apt-get install nfs-kernel-server 211 | $ sudo nano /etc/exports 212 | /var/lib/kubelet/migration/ 192.168.10.0/24(rw,sync,no_subtree_check) 213 | Note: 192.168.10.0/24 is subnetmask of every node in our cluster 214 | $ sudo exportfs -arvf 215 | $ sudo systemctl start nfs-kernel-server 216 | $ sudo systemctl enable nfs-kernel-server 217 | $ sudo systemctl status nfs-kernel-server 218 | $ sudo chmod 777 /var/lib/kubelet/migration 219 | ``` 220 | - Config NFS client at every worker nodes 221 | ``` 222 | $ sudo apt-get update 223 | $ sudo apt-get install nfs-common 224 | $ sudo nano /etc/fstab 225 | 192.168.10.13:/var/lib/kubelet/migration /var/lib/kubelet/migration nfs defaults,_netdev 0 0 226 | Note: 192.168.10.13 is the IP address of Nfs-server (master node) in this case. 227 | $ sudo umount /var/lib/kubelet/migration 228 | $ sudo mount -a 229 | (If Mount Error occured, $ mount -t nfs -o nfsvers=3 :/share /mnt) 230 | $ sudo chmod 777 /var/lib/kubelet/migration 231 | ``` 232 | - Ref: https://github.com/vutuong/personal-notes/blob/master/configNFS.md 233 | - You should remake the demo as the video in youtube after finish all the step above without```Step5-Approach 2```. 234 | - Only if you don't need to use pre-build binaries of kubelet and kubeadm or you need to edit kubelet source code by your self and rebuild: 235 | ### Step5- Approach 2: Download the custom source code and build. 236 | Download the custom source code and build at directory containerd/, the source code can be found as followings 237 | ``` 238 | $ git clone https://github.com/vutuong/kubernetes.git 239 | ``` 240 | You can find the binaries somewhere in the directories: ```kubernetes/kubernetes/_output/local/bin``` 241 | - The link ref for setting up environments and build the custom binaries: https://www.youtube.com/watch?v=Q91iZywBzew&t=3509s&ab_channel=CNCF%5BCloudNativeComputingFoundation%5D 242 | 243 | 244 | 245 | 246 | 247 | -------------------------------------------------------------------------------- /kubectl-plugin/README.md: -------------------------------------------------------------------------------- 1 | ### Checkpoint and migrate commands in Kubernetes 2 | In this repo, I write extensions for kubectl, which supports the commands of checkpoint and migrate a running pod in Kubernetes. 3 | ### How to build: 4 | * To build checkpoint command: 5 | ``` 6 | $ cd checkpoint-command 7 | $ go build -o kubectl-checkpoint 8 | $ sudo cp kubectl-checkpoint /usr/local/bin 9 | ``` 10 | * To build migrate command: 11 | ``` 12 | $ cd migrate-command 13 | $ go build -o kubectl-migrate 14 | $ sudo cp kubectl-migrate /usr/local/bin 15 | ``` 16 | ### How to use the checkpoint/migrate commands: 17 | * To run the checkpoint command: 18 | ``` 19 | $ kubectl checkpoint --help 20 | $ kubectl checkpoint [POD_NAME] [CHECKPOINT_PATH] 21 | ``` 22 | * Example of checkpoint a running pod: 23 | ``` 24 | dcn@dcn:~$ kubectl checkpoint --help 25 | 26 | checkpoint [POD_NAME] to [CHECKPOINT_PATH] 27 | 28 | Usage: 29 | checkpoint [OPTIONS] POD_NAME CHECKPOINT_PATH [flags] 30 | 31 | Examples: 32 | 33 | # Checkpoint a running Pod and save the checkpoint infomations to given path 34 | kubectl checkpoint [POD_NAME] [CHECKPOINT_PATH] 35 | kubectl checkpoint [POD_NAME] --namespace string [CHECKPOINT_PATH] 36 | 37 | 38 | Flags: 39 | -h, --help help for checkpoint 40 | --namespace string default namespace is "default" (default "default") 41 | ``` 42 | ``` 43 | dcn@dcn:~$ kubectl checkpoint simple /var/lib/kubelet/migration/xxx 44 | ``` 45 | * To run the migrate command: 46 | ``` 47 | dcn@dcn:~$ kubectl migrate --help 48 | dcn@dcn:~$ kubectl migrate video worker1 49 | ``` 50 | * Example of migrate a running pod: 51 | ``` 52 | dcn@dcn:~$ kubectl migrate --help 53 | 54 | migrate [POD_NAME] to [destHost] 55 | 56 | Usage: 57 | migrate [OPTIONS] POD_NAME destHost [flags] 58 | 59 | Examples: 60 | 61 | # Live-migrate a running Pod 62 | kubectl migrate [POD_NAME] [destHost] 63 | kubectl migrate [POD_NAME] --namespace string [destHost] 64 | 65 | 66 | Flags: 67 | -h, --help help for migrate 68 | --namespace string default namespace is "default" (default "default") 69 | ``` 70 | ``` 71 | dcn@dcn:~$ kubectl migrate video worker1 72 | response Status: 200 OK 73 | { 74 | "name": "video-migration-controller-70", 75 | "destHost": "", 76 | "replicas": 0, 77 | "selector": null, 78 | "action": "live-migration", 79 | "snapshotPath": "", 80 | "sourcePod": "video", 81 | "status": { 82 | "state": "", 83 | "currentRevision": "", 84 | "activePod": "" 85 | } 86 | } 87 | ``` 88 | -------------------------------------------------------------------------------- /kubectl-plugin/checkpoint-command/checkpoint_command.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path" 8 | "strings" 9 | "time" 10 | 11 | "github.com/spf13/cobra" 12 | corev1 "k8s.io/api/core/v1" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/client-go/kubernetes" 15 | "k8s.io/client-go/tools/clientcmd" 16 | ) 17 | 18 | /* To use this kubectl-plugin 19 | $ go build -o kubectl-checkpoint 20 | $ sudo cp kubectl-checkpoint /usr/local/bin 21 | */ 22 | 23 | const ( 24 | example = ` 25 | # Checkpoint a running Pod and save the checkpoint infomations to given path 26 | kubectl checkpoint [POD_NAME] [CHECKPOINT_PATH] 27 | kubectl checkpoint [POD_NAME] --namespace string [CHECKPOINT_PATH] 28 | ` 29 | longDesc = ` 30 | checkpoint [POD_NAME] to [CHECKPOINT_PATH] 31 | ` 32 | ) 33 | 34 | type MigrateArgs struct { 35 | 36 | // Pod select options 37 | Namespace string 38 | SourcePodName string 39 | checkpointPath string 40 | } 41 | 42 | func NewPluginCmd() *cobra.Command { 43 | var Margs MigrateArgs 44 | cmd := &cobra.Command{ 45 | Use: "checkpoint [OPTIONS] POD_NAME CHECKPOINT_PATH", 46 | Short: "checkpoint a Pod", 47 | Long: longDesc, 48 | Example: example, 49 | Run: func(c *cobra.Command, args []string) { 50 | if err := Margs.Complete(c, args); err != nil { 51 | fmt.Println(err) 52 | } 53 | /* 54 | if err := opts.Validate(); err != nil { 55 | fmt.Println(err) 56 | } 57 | if err := opts.Run(); err != nil { 58 | fmt.Println(err) 59 | } 60 | */ 61 | if err := Margs.Run(); err != nil { 62 | fmt.Println(err) 63 | } 64 | }, 65 | } 66 | cmd.Flags().StringVar(&Margs.Namespace, "namespace", "default", 67 | "default namespace is \"default\"") 68 | return cmd 69 | } 70 | 71 | func (a *MigrateArgs) Complete(cmd *cobra.Command, args []string) error { 72 | if len(args) == 0 { 73 | return fmt.Errorf("error pod not specified") 74 | } 75 | if len(args) == 1 { 76 | return fmt.Errorf("destHost not specified") 77 | } 78 | 79 | a.SourcePodName = args[0] 80 | a.checkpointPath = args[1] 81 | return nil 82 | } 83 | 84 | func (a *MigrateArgs) Run() error { 85 | // Step1: Get sourcePod 86 | ctx := context.Background() 87 | config, _ := clientcmd.BuildConfigFromFlags("", "/home/dcn/fault-detection/docs/anisble-playbook/kubernetes-the-hard-way/admin.kubeconfig") 88 | clientset, _ := kubernetes.NewForConfig(config) 89 | pod, _ := clientset.CoreV1().Pods("default").Get(ctx, a.SourcePodName, metav1.GetOptions{}) 90 | podsClient := clientset.CoreV1().Pods(corev1.NamespaceDefault) 91 | // Step2: Prepare the annotations 92 | action := "checkpoint" 93 | ann := pod.ObjectMeta.Annotations 94 | if ann == nil { 95 | ann = make(map[string]string) 96 | } 97 | ann["snapshotPolicy"] = action 98 | ann["snapshotPath"] = a.checkpointPath 99 | pod.ObjectMeta.Annotations = ann 100 | 101 | // Step3: Update the annotations 102 | if _, err := podsClient.Update(context.TODO(), pod, metav1.UpdateOptions{}); err != nil { 103 | return err 104 | } 105 | 106 | // Step4: Wait until checkpoint info are created 107 | 108 | container := pod.Spec.Containers[0].Name 109 | for { 110 | _, err := os.Stat(path.Join(a.checkpointPath, strings.Split(pod.Name, "-")[0], container, "descriptors.json")) 111 | if os.IsNotExist(err) { 112 | time.Sleep(1000 * time.Millisecond) 113 | } else { 114 | break 115 | } 116 | } 117 | // Step5: Stop checkpoint process 118 | ann["snapshotPolicy"] = "" 119 | ann["snapshotPath"] = "" 120 | pod.ObjectMeta.Annotations = ann 121 | if _, err := podsClient.Update(context.TODO(), pod, metav1.UpdateOptions{}); err != nil { 122 | return err 123 | } 124 | 125 | return nil 126 | } 127 | 128 | func main() { 129 | cmd := NewPluginCmd() 130 | cmd.Execute() 131 | } 132 | -------------------------------------------------------------------------------- /kubectl-plugin/checkpoint-command/kubectl-checkpoint: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/kubectl-plugin/checkpoint-command/kubectl-checkpoint -------------------------------------------------------------------------------- /kubectl-plugin/cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math/rand" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/spf13/cobra" 13 | // "k8s.io/client-go/kubernetes" 14 | // "k8s.io/client-go/tools/clientcmd" 15 | "sigs.k8s.io/controller-runtime/pkg/client" 16 | ) 17 | 18 | /* To use this kubectl-plugin 19 | $ go build -o kubectl-migrate 20 | $ sudo cp kubectl-migrate /usr/local/bin 21 | */ 22 | 23 | const ( 24 | example = ` 25 | # Checkpoint a Pod 26 | kubectl migrate POD_NAME destHost 27 | kubectl migrate POD_NAME --namespace string destHost 28 | ` 29 | longDesc = ` 30 | migrate POD_NAME to destHost 31 | ` 32 | ) 33 | 34 | type PodmigrationEndpoint struct { 35 | client client.Client 36 | } 37 | 38 | type MigrateArgs struct { 39 | 40 | // Pod select options 41 | Namespace string 42 | SourcePodName string 43 | DestHost string 44 | } 45 | 46 | type Podmigration struct { 47 | Name string `json:"name"` 48 | DestHost string `json:"destHost"` 49 | Replicas int `json:"replicas"` 50 | // Selector *metav1.LabelSelector `json:"selector"` 51 | Action string `json:"action"` 52 | SnapshotPath string `json:"snapshotPath"` 53 | SourcePod string `json:"sourcePod"` 54 | // Template corev1.PodTemplateSpec `json:"template"` 55 | } 56 | 57 | func NewPluginCmd() *cobra.Command { 58 | var Margs MigrateArgs 59 | cmd := &cobra.Command{ 60 | Use: "migrate [OPTIONS] POD_NAME destHost", 61 | Short: "migrate a Pod", 62 | Long: longDesc, 63 | Example: example, 64 | Run: func(c *cobra.Command, args []string) { 65 | if err := Margs.Complete(c, args); err != nil { 66 | fmt.Println(err) 67 | } 68 | if err := Margs.Run(); err != nil { 69 | fmt.Println(err) 70 | } 71 | }, 72 | } 73 | cmd.Flags().StringVar(&Margs.Namespace, "namespace", "default", 74 | "default namespace is \"default\"") 75 | return cmd 76 | } 77 | 78 | func (a *MigrateArgs) Complete(cmd *cobra.Command, args []string) error { 79 | if len(args) == 0 { 80 | return fmt.Errorf("error pod not specified") 81 | } 82 | if len(args) == 1 { 83 | return fmt.Errorf("destHost not specified") 84 | } 85 | 86 | a.SourcePodName = args[0] 87 | a.DestHost = args[1] 88 | return nil 89 | } 90 | 91 | func (a *MigrateArgs) Run() error { 92 | // ctx := context.Background() 93 | //read the config file, so the plugin can talk to API-server 94 | // config, _ := clientcmd.BuildConfigFromFlags("", "/home/dcn/fault-detection/docs/anisble-playbook/kubernetes-the-hard-way/admin.kubeconfig") 95 | // clientset, _ := kubernetes.NewForConfig(config) 96 | // pod, _ := clientset.CoreV1().Pods("default").Get(ctx, "pingpod-worker1", metav1.GetOptions{}) 97 | url := "http://localhost:5000/Podmigrations" 98 | method := "POST" 99 | crdName := a.getCrdName() 100 | action := "live-migration" 101 | sourcePod := a.SourcePodName 102 | destHost := a.DestHost 103 | data := fmt.Sprintf(`{"name": "%s","action": "%s","sourcePod": "%s","destHost":"%s"}`, crdName, action, sourcePod, destHost) 104 | payload := strings.NewReader(data) 105 | client := &http.Client{} 106 | req, err := http.NewRequest(method, url, payload) 107 | if err != nil { 108 | fmt.Println(err) 109 | } 110 | req.Header.Add("contentType", "application/json") 111 | req.Header.Add("Content-Type", "application/json") 112 | 113 | res, err := client.Do(req) 114 | 115 | body, err := ioutil.ReadAll(res.Body) 116 | fmt.Println("response Status:", res.Status) 117 | fmt.Println(string(body)) 118 | fmt.Println("testing .............") 119 | if err != nil { 120 | return err 121 | } 122 | defer res.Body.Close() 123 | return nil 124 | } 125 | 126 | func (a *MigrateArgs) getCrdName() string { 127 | s1 := rand.NewSource(time.Now().UnixNano()) 128 | number := rand.New(s1) 129 | crdName := a.SourcePodName + "-migration-controller-" + strconv.Itoa(number.Intn(100)) 130 | return crdName 131 | } 132 | 133 | func main() { 134 | cmd := NewPluginCmd() 135 | cmd.Execute() 136 | } 137 | -------------------------------------------------------------------------------- /kubectl-plugin/migrate-command/kubectl-migrate: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/kubectl-plugin/migrate-command/kubectl-migrate -------------------------------------------------------------------------------- /kubectl-plugin/migrate-command/migrate_command.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math/rand" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/spf13/cobra" 13 | // "k8s.io/client-go/kubernetes" 14 | // "k8s.io/client-go/tools/clientcmd" 15 | "sigs.k8s.io/controller-runtime/pkg/client" 16 | ) 17 | 18 | /* To use this kubectl-plugin 19 | $ go build -o kubectl-migrate 20 | $ sudo cp kubectl-migrate /usr/local/bin 21 | */ 22 | 23 | const ( 24 | example = ` 25 | # Live-migrate a running Pod 26 | kubectl migrate [POD_NAME] [destHost] 27 | kubectl migrate [POD_NAME] --namespace string [destHost] 28 | ` 29 | longDesc = ` 30 | migrate [POD_NAME] to [destHost] 31 | ` 32 | ) 33 | 34 | type PodmigrationEndpoint struct { 35 | client client.Client 36 | } 37 | 38 | type MigrateArgs struct { 39 | 40 | // Pod select options 41 | Namespace string 42 | SourcePodName string 43 | DestHost string 44 | } 45 | 46 | type Podmigration struct { 47 | Name string `json:"name"` 48 | DestHost string `json:"destHost"` 49 | Replicas int `json:"replicas"` 50 | // Selector *metav1.LabelSelector `json:"selector"` 51 | Action string `json:"action"` 52 | SnapshotPath string `json:"snapshotPath"` 53 | SourcePod string `json:"sourcePod"` 54 | // Template corev1.PodTemplateSpec `json:"template"` 55 | } 56 | 57 | func NewPluginCmd() *cobra.Command { 58 | var Margs MigrateArgs 59 | cmd := &cobra.Command{ 60 | Use: "migrate [OPTIONS] POD_NAME destHost", 61 | Short: "migrate a Pod", 62 | Long: longDesc, 63 | Example: example, 64 | Run: func(c *cobra.Command, args []string) { 65 | if err := Margs.Complete(c, args); err != nil { 66 | fmt.Println(err) 67 | } 68 | /* 69 | if err := opts.Validate(); err != nil { 70 | fmt.Println(err) 71 | } 72 | if err := opts.Run(); err != nil { 73 | fmt.Println(err) 74 | } 75 | */ 76 | if err := Margs.Run(); err != nil { 77 | fmt.Println(err) 78 | } 79 | }, 80 | } 81 | cmd.Flags().StringVar(&Margs.Namespace, "namespace", "default", 82 | "default namespace is \"default\"") 83 | return cmd 84 | } 85 | 86 | func (a *MigrateArgs) Complete(cmd *cobra.Command, args []string) error { 87 | if len(args) == 0 { 88 | return fmt.Errorf("error pod not specified") 89 | } 90 | if len(args) == 1 { 91 | return fmt.Errorf("destHost not specified") 92 | } 93 | 94 | a.SourcePodName = args[0] 95 | a.DestHost = args[1] 96 | return nil 97 | } 98 | 99 | func (a *MigrateArgs) Run() error { 100 | // ctx := context.Background() 101 | //read the config file, so the plugin can talk to API-server 102 | // config, _ := clientcmd.BuildConfigFromFlags("", "/home/dcn/fault-detection/docs/anisble-playbook/kubernetes-the-hard-way/admin.kubeconfig") 103 | // clientset, _ := kubernetes.NewForConfig(config) 104 | // pod, _ := clientset.CoreV1().Pods("default").Get(ctx, "pingpod-worker1", metav1.GetOptions{}) 105 | 106 | // Step1: Define argument 107 | url := "http://localhost:5000/Podmigrations" 108 | method := "POST" 109 | crdName := a.getCrdName() 110 | action := "live-migration" 111 | sourcePod := a.SourcePodName 112 | destHost := a.DestHost 113 | data := fmt.Sprintf(`{"name": "%s","action": "%s","sourcePod": "%s","destHost":"%s"}`, crdName, action, sourcePod, destHost) 114 | payload := strings.NewReader(data) 115 | client := &http.Client{} 116 | req, err := http.NewRequest(method, url, payload) 117 | if err != nil { 118 | fmt.Println(err) 119 | } 120 | req.Header.Add("contentType", "application/json") 121 | req.Header.Add("Content-Type", "application/json") 122 | res, err := client.Do(req) 123 | body, err := ioutil.ReadAll(res.Body) 124 | fmt.Println("response Status:", res.Status) 125 | fmt.Println(string(body)) 126 | if err != nil { 127 | return err 128 | } 129 | defer res.Body.Close() 130 | return nil 131 | } 132 | 133 | func (a *MigrateArgs) getCrdName() string { 134 | s1 := rand.NewSource(time.Now().UnixNano()) 135 | number := rand.New(s1) 136 | crdName := a.SourcePodName + "-migration-controller-" + strconv.Itoa(number.Intn(100)) 137 | return crdName 138 | } 139 | 140 | func main() { 141 | cmd := NewPluginCmd() 142 | cmd.Execute() 143 | } 144 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | "k8s.io/apimachinery/pkg/runtime" 24 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 25 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 28 | 29 | podmigv1 "github.com/SSU-DCN/podmigration-operator/api/v1" 30 | "github.com/SSU-DCN/podmigration-operator/controllers" 31 | // +kubebuilder:scaffold:imports 32 | ) 33 | 34 | var ( 35 | scheme = runtime.NewScheme() 36 | setupLog = ctrl.Log.WithName("setup") 37 | ) 38 | 39 | func init() { 40 | _ = clientgoscheme.AddToScheme(scheme) 41 | 42 | _ = podmigv1.AddToScheme(scheme) 43 | // +kubebuilder:scaffold:scheme 44 | } 45 | 46 | func main() { 47 | var metricsAddr string 48 | var enableLeaderElection bool 49 | flag.StringVar(&metricsAddr, "metrics-addr", ":8081", "The address the metric endpoint binds to.") 50 | flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, 51 | "Enable leader election for controller manager. "+ 52 | "Enabling this will ensure there is only one active controller manager.") 53 | flag.Parse() 54 | 55 | ctrl.SetLogger(zap.New(zap.UseDevMode(true))) 56 | 57 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 58 | Scheme: scheme, 59 | MetricsBindAddress: metricsAddr, 60 | Port: 9443, 61 | LeaderElection: enableLeaderElection, 62 | LeaderElectionID: "a9ee352c.dcn.ssu.ac.kr", 63 | }) 64 | if err != nil { 65 | setupLog.Error(err, "unable to start manager") 66 | os.Exit(1) 67 | } 68 | 69 | if err = (&controllers.PodmigrationReconciler{ 70 | Client: mgr.GetClient(), 71 | Log: ctrl.Log.WithName("controllers").WithName("Podmigration"), 72 | Scheme: mgr.GetScheme(), 73 | }).SetupWithManager(mgr); err != nil { 74 | setupLog.Error(err, "unable to create controller", "controller", "Podmigration") 75 | os.Exit(1) 76 | } 77 | // +kubebuilder:scaffold:builder 78 | 79 | setupLog.Info("starting manager") 80 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 81 | setupLog.Error(err, "problem running manager") 82 | os.Exit(1) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /podmigration.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SSU-DCN/podmigration-operator/2c421a073d98ff4f5c547c8d079ad289c341c42c/podmigration.jpg --------------------------------------------------------------------------------