├── .github └── workflows │ ├── ci-pr.yaml │ └── ci-release.yaml ├── .gitignore ├── Dockerfile ├── PAGINATE_SPEC.md ├── README.md ├── api ├── extend │ └── deploy2service.go ├── proxy.go ├── proxy_test.go └── req.go ├── cmd ├── cacheproxy │ └── main.go ├── ckube-plugin │ └── main.go └── ckubecli │ └── main.go ├── common ├── config.go └── constants │ └── ckube.go ├── config └── example.json ├── entrypoint.sh ├── examples ├── paginate │ └── main.go └── search │ └── main.go ├── go.mod ├── go.sum ├── kube ├── kube.go └── label.go ├── log └── log.go ├── nginx.conf ├── page ├── page.go └── page_test.go ├── pkg └── client │ └── fake │ ├── fake_ckube.go │ ├── fake_client_test.go │ └── interface.go ├── server ├── route.go └── server.go ├── store ├── interface.go ├── memory │ ├── memory.go │ └── memory_test.go └── model.go ├── utils ├── fs_watcher.go ├── prommonitor │ └── metrics.go ├── subset.go ├── subset_test.go └── utils.go └── watcher ├── interface.go └── watcher.go /.github/workflows/ci-pr.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | style-check: 11 | runs-on: ubuntu-20.04 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Set up Go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: 1.19 18 | - name: golangci-lint 19 | if: github.event_name == 'pull_request' 20 | uses: golangci/golangci-lint-action@v2 21 | with: 22 | version: v1.49.0 23 | go-unit-test: 24 | runs-on: ubuntu-20.04 25 | needs: [style-check] 26 | steps: 27 | - uses: actions/checkout@v3 28 | - name: Set up Go 29 | uses: actions/setup-go@v2 30 | with: 31 | go-version: 1.19 32 | - name: Test 33 | run: go test -v ./... 34 | docker-build: 35 | runs-on: ubuntu-latest 36 | needs: [go-unit-test] 37 | if: github.ref_name == 'main' 38 | steps: 39 | - name: Checkout 40 | uses: actions/checkout@v3 41 | - name: Set up QEMU 42 | uses: docker/setup-qemu-action@v1 43 | - name: Set up Docker Buildx 44 | uses: docker/setup-buildx-action@v1 45 | - name: Login Github Container registry 46 | uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 47 | with: 48 | registry: ghcr.io 49 | username: ${{ github.actor }} 50 | password: ${{ secrets.GITHUB_TOKEN }} 51 | 52 | - name: Docker meta 53 | id: meta 54 | uses: docker/metadata-action@v3 55 | with: 56 | images: | 57 | ghcr.io/daocloud/ckube 58 | tags: | 59 | type=raw,value=latest 60 | 61 | - name: Build 62 | uses: docker/build-push-action@v2 63 | with: 64 | context: . 65 | platforms: linux/amd64,linux/arm64 66 | push: true 67 | tags: ${{ steps.meta.outputs.tags }} 68 | labels: ${{ steps.meta.outputs.labels }} 69 | cache-from: type=gha 70 | cache-to: type=gha,mode=max 71 | -------------------------------------------------------------------------------- /.github/workflows/ci-release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | docker-build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | - name: Set up QEMU 15 | uses: docker/setup-qemu-action@v1 16 | - name: Set up Docker Buildx 17 | uses: docker/setup-buildx-action@v1 18 | - name: Login Github Container registry 19 | uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 20 | with: 21 | registry: ghcr.io 22 | username: ${{ github.actor }} 23 | password: ${{ secrets.GITHUB_TOKEN }} 24 | 25 | - name: Docker meta 26 | id: meta 27 | uses: docker/metadata-action@v3 28 | with: 29 | images: | 30 | ghcr.io/daocloud/ckube 31 | tags: | 32 | type=semver,pattern={{version}} 33 | 34 | - name: Build 35 | uses: docker/build-push-action@v2 36 | with: 37 | context: . 38 | platforms: linux/amd64,linux/arm64 39 | push: true 40 | tags: ${{ steps.meta.outputs.tags }} 41 | labels: ${{ steps.meta.outputs.labels }} 42 | cache-from: type=gha 43 | cache-to: type=gha,mode=max 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ---> Go 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | 18 | .idea 19 | .vscode 20 | 21 | # logs 22 | _log 23 | 24 | *.local.yaml 25 | .DS_Store 26 | _output 27 | config/local.json 28 | examples/local/* 29 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ###### build stage #### 2 | FROM golang:1.19 as build 3 | 4 | WORKDIR /app 5 | 6 | RUN wget https://storage.googleapis.com/kubernetes-release/release/v1.23.3/bin/linux/$(uname -m | sed -E 's/x86_64/amd64/g' | sed -E 's/aarch64/arm64/g')/kubectl -O /usr/bin/kubectl && \ 7 | chmod +x /usr/bin/kubectl 8 | 9 | ADD go.mod . 10 | ADD go.sum . 11 | 12 | RUN go mod download 13 | 14 | ADD . . 15 | 16 | RUN go build -ldflags "-s -w" -o ./dist/cacheproxy ./cmd/cacheproxy/main.go && \ 17 | go build -ldflags "-s -w" -o ./dist/kubectl-ckube ./cmd/ckube-plugin/main.go 18 | 19 | FROM ubuntu:20.04 20 | 21 | WORKDIR /app 22 | 23 | COPY --from=build /usr/bin/kubectl /usr/local/bin/ 24 | COPY --from=build /app/dist/kubectl-ckube /usr/local/bin/ 25 | COPY --from=build /app/dist/cacheproxy dist/cacheproxy 26 | 27 | CMD ["/app/dist/cacheproxy"] 28 | -------------------------------------------------------------------------------- /PAGINATE_SPEC.md: -------------------------------------------------------------------------------- 1 | # Paginate Spec 2 | 3 | 本文描述 CKube 的查询、分页功能定义。 4 | 5 | ## Page 6 | Page 表示页码，从 1 开始，为 0 表示不分页。小于 0 为非法。 7 | 8 | ## PageSize 9 | PageSize 表示每页数量，为 0 表示不分页，小于 0 为非法。 10 | 11 | ## Search 12 | 13 | ### Search DSL 14 | CKube 的搜索支持模糊搜索（Fuzzy Search）,精准匹配和高级搜索。 15 | 如果同时有模糊搜索、精准字段匹配或高级搜索的多个，以 `;` 分割，如果需要搜索 `;`，请使用两个： `;;`。 16 | 如 `name=e; __ckube_as__:namespaces in (default, test)` 17 | 18 | #### 模糊搜索 Fuzzy Search 19 | 模糊搜索的句式为任意字符串，但是中间不能包含 `=` 和 `!`. 20 | 模糊搜索的算法为，只要`任意索引中的值包含`此项，即为匹配，会返回此结果。 21 | 以 `!` 开头，表示不匹配。 22 | 23 | #### 精准字段匹配 24 | 精准匹配的句式为 `key=value`，其中，`key` 为索引的 Key，`value` 为需要匹配的内容。 25 | `key` 需要满足 `[\d\w\-_\.]` 的规则，`value` 可以是任意值。 26 | 精准匹配的算法为，直接对索引的键值做匹配，只要索引中的值包含 `value` 即为匹配成功，如果要精准匹配，可将 `value` 加上双引号，如 `name="test"`。`value` 以 `!` 开头表示不匹配。 27 | 28 | #### 高级搜索 29 | 主要针对需要同时对多个字段进行匹配，或者需要执行 `in`, `notin`, `!=` 等操作。 30 | 高级搜索的句式为 `__ckube_as__:`, `LABEL_SELECTOR` 的语法参考：https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors 31 | 大致总结如下： 32 | 33 | | 需求 | 样例 | 34 | | -- | -- | 35 | | 精确匹配某个字段 | `__ckube_as__:name=ok` | 36 | | 不匹配 | `__ckube_as__:name!=ok` | 37 | | 匹配列表 | `__ckube_as__:name in (a,b,c)` | 38 | | 不匹配列表 | `__ckube_as__:name notin (a,b,c)` | 39 | | 多个条件 | `__ckube_as__:name=ok,sp notin (ok, 1)` | 40 | 41 | 高级搜索会按照规则对语句进行解析，然后逐个匹配。 42 | 43 | ## Sort 44 | 45 | Sort 用于对结果进行排序，CKube 支持同时对多个字段进行排序，并且支持`字符串`和`数字`类型的字段进行排序。 46 | 47 | ### Sort DSL 48 | 49 | 搜索的句式为 `key[!int|!str][ desc|asc][,key[!int|!str][ desc|asc]...]`. 50 | `[]` 表示可选。 51 | `!int` 如果存在，表示对字段进行强制数字转换，默认使用字符串排序规则进行排序。 52 | `desc` 表示使用该字段进行反向排序，`asc` 表示对该字段进行正向排序。 53 | 如果排序字段为空，将使用 uid,name 优先的字段进行排序。 54 | 55 | #### 样例 56 | 按照命名空间，名字进行排序 `namespace,name`. 57 | 按照命名空间反序，副本数进行排序 `namespace desc,replicas!int`. 58 | 按照创建时间进行排序 `createTimestamp!int desc`. 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ckube 2 | 3 | Kubernetes APIServer 高性能代理组件，代理 APIServer 的 List 请求，其它类型的请求会直接反向代理到原生 APIServer。 4 | CKube 还额外支持了分页、搜索和索引等功能。 5 | 并且，CKube 100% 兼容原生 kubectl 和 kube client sdk，只需要简单的配置即可实现全局替换。 6 | 7 | ## 安装部署 8 | 9 | 使用 CKube 构建好的镜像直接启动即可，建议直接部署在 Kubernetes 集群里面，为其绑定 ClusterRole 并赋予权限。 10 | 样例 YAML： 11 | 12 | ```yaml 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | kind: ClusterRole 15 | metadata: 16 | name: ckube 17 | labels: 18 | app: ckube 19 | rules: 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - pods 24 | - services 25 | - configmaps 26 | - events 27 | - namespaces 28 | verbs: 29 | - get 30 | - list 31 | - watch 32 | - create 33 | - update 34 | - patch 35 | --- 36 | apiVersion: rbac.authorization.k8s.io/v1 37 | kind: ClusterRoleBinding 38 | metadata: 39 | name: ckube 40 | labels: 41 | app: ckube 42 | roleRef: 43 | apiGroup: rbac.authorization.k8s.io 44 | kind: ClusterRole 45 | name: ckube 46 | subjects: 47 | - kind: ServiceAccount 48 | name: ckube 49 | namespace: kube-system 50 | --- 51 | apiVersion: v1 52 | kind: ServiceAccount 53 | metadata: 54 | name: ckube 55 | namespace: kube-system 56 | labels: 57 | app: ckube 58 | --- 59 | apiVersion: v1 60 | kind: Service 61 | metadata: 62 | name: ckube 63 | labels: 64 | app: ckube 65 | spec: 66 | type: ClusterIP 67 | ports: 68 | - port: 80 69 | protocol: TCP 70 | name: http-ckube 71 | selector: 72 | app: ckube 73 | 74 | --- 75 | apiVersion: apps/v1 76 | kind: Deployment 77 | metadata: 78 | name: ckube 79 | labels: 80 | app: ckube 81 | spec: 82 | replicas: 1 83 | selector: 84 | matchLabels: 85 | app: ckube 86 | template: 87 | metadata: 88 | labels: 89 | app: ckube 90 | spec: 91 | dnsPolicy: ClusterFirst 92 | containers: 93 | - name: ckube 94 | image: registry.daocloud.cn/mesh/dx-mesh-ckube:0.0.0-1094 95 | volumeMounts: 96 | - readOnly: true 97 | mountPath: /app/config 98 | name: ckube-config 99 | serviceAccountName: ckube 100 | volumes: 101 | - name: ckube-config 102 | configMap: 103 | name: ckube 104 | --- 105 | apiVersion: v1 106 | kind: ConfigMap 107 | metadata: 108 | name: ckube 109 | labels: 110 | app: ckube 111 | data: 112 | local.json: |- 113 | { 114 | "proxies": [ 115 | { 116 | "group": "", 117 | "version": "v1", 118 | "resource": "pods", 119 | "list_kind": "PodList", 120 | "index": { 121 | "namespace": "{.metadata.namespace}", 122 | "name": "{.metadata.name}", 123 | "labels": "{.metadata.labels}", 124 | "created_at": "{.metadata.creationTimestamp}" 125 | } 126 | }, 127 | { 128 | "group": "", 129 | "version": "v1", 130 | "resource": "services", 131 | "list_kind": "ServiceList", 132 | "index": { 133 | "namespace": "{.metadata.namespace}", 134 | "name": "{.metadata.name}", 135 | "labels": "{.metadata.labels}", 136 | "created_at": "{.metadata.creationTimestamp}" 137 | } 138 | }, 139 | { 140 | "group": "", 141 | "version": "v1", 142 | "resource": "namespaces", 143 | "list_kind": "NamespaceList", 144 | "index": { 145 | "namespace": "{.metadata.namespace}", 146 | "name": "{.metadata.name}", 147 | "labels": "{.metadata.labels}", 148 | "created_at": "{.metadata.creationTimestamp}" 149 | } 150 | } 151 | ] 152 | } 153 | 154 | ``` 155 | 156 | ## 使用方法 157 | 158 | 因为 CKube 原生兼容 APIServer 接口，所以，不需要使用额外的 SDK 或切换目前项目中的用法。 159 | 如果再程序中需要使用 CKube 来提升性能，或者需要实现分页、搜索等功能，只需要在 SDK 初始化的时候，将地址指定为部署好的 CKube 地址即可。 160 | 详细使用方法可以参考 `examples` 目录下的方法。 161 | 162 | ## 配置方法 163 | 164 | 参考 `config/example.json` 文件进行配置。 165 | 对于每一个需要加速的资源，都需要在配置文件中进行定义，不然无法实现加速和分页等功能。 166 | 167 | -------------------------------------------------------------------------------- /api/extend/deploy2service.go: -------------------------------------------------------------------------------- 1 | package extend 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/gorilla/mux" 7 | v1 "k8s.io/api/core/v1" 8 | v12 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/util/json" 10 | 11 | "github.com/DaoCloud/ckube/api" 12 | "github.com/DaoCloud/ckube/common" 13 | "github.com/DaoCloud/ckube/page" 14 | "github.com/DaoCloud/ckube/store" 15 | "github.com/DaoCloud/ckube/utils" 16 | "github.com/DaoCloud/ckube/watcher" 17 | ) 18 | 19 | func Deploy2Service(r *api.ReqContext) interface{} { 20 | cluster := mux.Vars(r.Request)["cluster"] 21 | ns := mux.Vars(r.Request)["namespace"] 22 | dep := mux.Vars(r.Request)["deployment"] 23 | services := []*v1.Service{} 24 | podGvr := store.GroupVersionResource{ 25 | Group: "", 26 | Version: "v1", 27 | Resource: "pods", 28 | } 29 | svcGvr := store.GroupVersionResource{ 30 | Group: "", 31 | Version: "v1", 32 | Resource: "services", 33 | } 34 | if cluster == "" { 35 | cluster = common.GetConfig().DefaultCluster 36 | } 37 | p := page.Paginate{Search: "name=" + dep} 38 | _ = p.Clusters([]string{cluster}) 39 | res := r.Store.Query(podGvr, store.Query{ 40 | Namespace: ns, 41 | Paginate: p, 42 | }) 43 | if res.Error != nil { 44 | return res.Error 45 | } 46 | var labels map[string]string 47 | for _, podIf := range res.Items { 48 | if pod, ok := podIf.(v12.Object); ok { 49 | depName := getDeploymentName(pod) 50 | if depName != "" { 51 | labels = pod.GetLabels() 52 | break 53 | } 54 | } 55 | } 56 | p = page.Paginate{} 57 | _ = p.Clusters([]string{cluster}) 58 | res = r.Store.Query(svcGvr, store.Query{ 59 | Namespace: ns, 60 | Paginate: p, 61 | }) 62 | if res.Error != nil { 63 | return res.Error 64 | } 65 | for _, svcIf := range res.Items { 66 | svc := &v1.Service{} 67 | if s, ok := svcIf.(*watcher.ObjType); ok { 68 | bs, _ := json.Marshal(s) 69 | _ = json.Unmarshal(bs, svc) 70 | } else { 71 | svc = svcIf.(*v1.Service) 72 | } 73 | if svc.Spec.Selector != nil && utils.IsSubsetOf(svc.Spec.Selector, labels) { 74 | services = append(services, svc) 75 | } 76 | } 77 | return services 78 | } 79 | 80 | func getDeploymentName(pod v12.Object) string { 81 | if len(pod.GetOwnerReferences()) == 0 || pod.GetOwnerReferences()[0].Kind != "ReplicaSet" { 82 | return "" 83 | } else { 84 | parts := strings.Split(pod.GetOwnerReferences()[0].Name, "-") 85 | return strings.Join(parts[:len(parts)-1], "-") 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /api/proxy.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/base64" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "reflect" 12 | "sort" 13 | "strings" 14 | "time" 15 | 16 | "github.com/gorilla/mux" 17 | "k8s.io/apimachinery/pkg/api/errors" 18 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | k8labels "k8s.io/apimachinery/pkg/labels" 20 | "k8s.io/apimachinery/pkg/types" 21 | "k8s.io/client-go/rest" 22 | 23 | "github.com/DaoCloud/ckube/common" 24 | "github.com/DaoCloud/ckube/common/constants" 25 | "github.com/DaoCloud/ckube/kube" 26 | "github.com/DaoCloud/ckube/log" 27 | "github.com/DaoCloud/ckube/page" 28 | "github.com/DaoCloud/ckube/store" 29 | ) 30 | 31 | func getGVRFromReq(req *http.Request) store.GroupVersionResource { 32 | group := mux.Vars(req)["group"] 33 | version := mux.Vars(req)["version"] 34 | resourceType := mux.Vars(req)["resourceType"] 35 | 36 | gvr := store.GroupVersionResource{ 37 | Group: group, 38 | Version: version, 39 | Resource: resourceType, 40 | } 41 | return gvr 42 | } 43 | 44 | func findLabels(i interface{}) map[string]string { 45 | meta := reflect.ValueOf(i).Elem().FieldByName("ObjectMeta") 46 | if !meta.CanInterface() { 47 | meta = reflect.ValueOf(i).Elem().FieldByName("metadata") 48 | if !meta.CanInterface() { 49 | return nil 50 | } 51 | } 52 | metaInterface := meta.Interface() 53 | labels := reflect.ValueOf(metaInterface).FieldByName("Labels") 54 | if !labels.CanInterface() { 55 | labels = reflect.ValueOf(metaInterface).FieldByName("labels") 56 | if !labels.CanInterface() { 57 | return nil 58 | } 59 | } 60 | res := labels.Interface().(map[string]string) 61 | return res 62 | } 63 | 64 | func errorProxy(w http.ResponseWriter, err v1.Status) interface{} { 65 | w.Header().Set("Content-Type", "application/json") 66 | w.WriteHeader(int(err.Code)) 67 | err.Kind = "Status" 68 | err.APIVersion = "v1" 69 | return err 70 | } 71 | 72 | func ProxySingleResources(r *ReqContext, gvr store.GroupVersionResource, cluster, namespace, resource string) interface{} { 73 | res := r.Store.Get(gvr, cluster, namespace, resource) 74 | if res == nil { 75 | return errorProxy(r.Writer, v1.Status{ 76 | Status: v1.StatusFailure, 77 | Message: fmt.Sprintf("resource %v: %s/%s/%s not found", gvr, cluster, namespace, resource), 78 | Reason: v1.StatusReasonNotFound, 79 | Details: nil, 80 | Code: 404, 81 | }) 82 | } 83 | return res 84 | } 85 | 86 | type bytesBody struct { 87 | io.Reader 88 | } 89 | 90 | func (b *bytesBody) Close() error { 91 | return nil 92 | } 93 | 94 | func wrapReader(reader io.Reader) io.ReadCloser { 95 | return &bytesBody{reader} 96 | } 97 | 98 | func parsePaginateAndLabelsAndClean(r *http.Request) (*page.Paginate, *v1.LabelSelector, string, error) { 99 | var labels *v1.LabelSelector 100 | var paginate page.Paginate 101 | var labelSelectorStr string 102 | clusterPrefix := constants.ClusterPrefix 103 | cluster := "" 104 | query := r.URL.Query() 105 | for k, v := range query { 106 | switch k { 107 | case "labelSelector": // For List options 108 | labelSelectorStr = v[0] 109 | case "cluster": 110 | cluster = v[0] 111 | case "fieldManager", "resourceVersion": // For Get Create Patch Update actions. 112 | if strings.HasPrefix(v[0], clusterPrefix) { 113 | cluster = v[0][len(clusterPrefix):] 114 | query.Del(k) 115 | } 116 | } 117 | } 118 | if r.Method == http.MethodDelete { 119 | body := r.Body 120 | opts, err := io.ReadAll(body) 121 | if err == nil { 122 | options := v1.DeleteOptions{} 123 | _ = json.Unmarshal(opts, &options) 124 | if len(options.DryRun) > 0 && strings.HasPrefix(options.DryRun[0], clusterPrefix) { 125 | cluster = options.DryRun[0][len(clusterPrefix):] 126 | options.DryRun = options.DryRun[1:] 127 | bs, _ := json.Marshal(options) 128 | r.Body = wrapReader(bytes.NewBuffer(bs)) 129 | } 130 | } else { 131 | log.Warnf("read body error: %v", err) 132 | } 133 | } 134 | if labelSelectorStr != "" { 135 | var err error 136 | labels, err = kube.ParseToLabelSelector(labelSelectorStr) 137 | if err != nil { 138 | return nil, nil, cluster, err 139 | } 140 | paginateStr := "" 141 | if ps, ok := labels.MatchLabels[constants.PaginateKey]; ok { 142 | paginateStr = ps 143 | delete(labels.MatchLabels, constants.PaginateKey) 144 | } else { 145 | mes := []v1.LabelSelectorRequirement{} 146 | // Why we use MatchExpressions? 147 | // to adapt dsm.daocloud.io/query=xxxx send to apiserver, which makes no results. 148 | // if dsm.daocloud.io/query != xxx or dsm.daocloud.io/query not in (xxx), results exist even if it was sent to apiserver. 149 | for _, m := range labels.MatchExpressions { 150 | if m.Key == constants.PaginateKey { 151 | if len(m.Values) > 0 { 152 | paginateStr, err = kube.MergeValues(m.Values) 153 | if err != nil { 154 | return nil, labels, cluster, err 155 | } 156 | } 157 | } else { 158 | mes = append(mes, m) 159 | } 160 | } 161 | labels.MatchExpressions = mes 162 | } 163 | if paginateStr != "" { 164 | rr, err := base64.StdEncoding.WithPadding(base64.NoPadding).DecodeString(paginateStr) 165 | if err != nil { 166 | return nil, labels, cluster, err 167 | } 168 | _ = json.Unmarshal(rr, &paginate) 169 | delete(labels.MatchLabels, constants.PaginateKey) 170 | } 171 | if len(labels.MatchLabels) != 0 || len(labels.MatchExpressions) != 0 { 172 | // if labelSelectorStr is empty, v1.FormatLabelSelector will return "", so we should not use it. 173 | query.Set("labelSelector", v1.FormatLabelSelector(labels)) 174 | } 175 | } 176 | r.URL.RawQuery = query.Encode() 177 | if cs := paginate.GetClusters(); len(cs) > 0 && cluster == "" { 178 | cluster = cs[0] 179 | } 180 | if cluster == "" { 181 | cluster = common.GetConfig().DefaultCluster 182 | } 183 | return &paginate, labels, cluster, nil 184 | } 185 | 186 | func Proxy(r *ReqContext) interface{} { 187 | // version := mux.Vars(r.Request)["version"] 188 | namespace := mux.Vars(r.Request)["namespace"] 189 | resourceName := mux.Vars(r.Request)["resource"] 190 | gvr := getGVRFromReq(r.Request) 191 | paginate, labels, cluster, err := parsePaginateAndLabelsAndClean(r.Request) 192 | if err != nil { 193 | return proxyPass(r, cluster) 194 | } 195 | if cluster == "" { 196 | cluster = common.GetConfig().DefaultCluster 197 | } 198 | for k, v := range r.Request.URL.Query() { 199 | switch k { 200 | case "labelSelector": 201 | case "timeoutSeconds": 202 | case "timeout": 203 | case "limit": 204 | default: 205 | log.Warnf("got unexpected query key: %s, value: %v, proxyPass to api server", k, v) 206 | return proxyPass(r, cluster) 207 | } 208 | } 209 | if paginate == nil { 210 | paginate = &page.Paginate{} 211 | } 212 | if !r.Store.IsStoreGVR(gvr) || r.Request.Method != "GET" { 213 | log.Debugf("gvr %v no cached or method not GET", gvr) 214 | return proxyPass(r, cluster) 215 | } 216 | if resourceName != "" { 217 | return ProxySingleResources(r, gvr, cluster, namespace, resourceName) 218 | } 219 | // default only get default cluster's resources, 220 | // If you want to get all clusters' resources, 221 | // please call paginate.Clusters() before fetch resources 222 | if cs := paginate.GetClusters(); len(cs) == 0 { 223 | err = paginate.Clusters([]string{common.GetConfig().DefaultCluster}) 224 | if err != nil { 225 | log.Errorf("set cluster error: %v", err) 226 | } 227 | } 228 | log.Debugf("got paginate %v", paginate) 229 | 230 | items := make([]interface{}, 0) 231 | var total int64 232 | if labels != nil && (len(labels.MatchLabels) != 0 || len(labels.MatchExpressions) != 0) { 233 | // exists label selector 234 | res := r.Store.Query(gvr, store.Query{ 235 | Namespace: namespace, 236 | Paginate: page.Paginate{ 237 | Sort: paginate.Sort, 238 | Search: paginate.Search, 239 | }, // get all 240 | }) 241 | if res.Error != nil { 242 | return errorProxy(r.Writer, v1.Status{ 243 | Status: v1.StatusFailure, 244 | Message: "query error", 245 | Reason: v1.StatusReason(res.Error.Error()), 246 | Code: 400, 247 | }) 248 | } 249 | sel, err := v1.LabelSelectorAsSelector(labels) 250 | if err != nil { 251 | return errorProxy(r.Writer, v1.Status{ 252 | Status: v1.StatusFailure, 253 | Message: "label selector parse error", 254 | Reason: v1.StatusReason(err.Error()), 255 | Details: nil, 256 | Code: 400, 257 | }) 258 | } 259 | for _, item := range res.Items { 260 | l := findLabels(item) 261 | if sel.Matches(k8labels.Set(l)) { 262 | items = append(items, item) 263 | } 264 | } 265 | 266 | // manually slice items 267 | var l = int64(len(items)) 268 | var start, end int64 269 | if paginate.PageSize == 0 || paginate.Page == 0 { 270 | // all resources 271 | start = 0 272 | end = l 273 | } else { 274 | start = (paginate.Page - 1) * paginate.PageSize 275 | end = start + paginate.PageSize 276 | if start >= l { 277 | start = l 278 | } 279 | if end >= l { 280 | end = l 281 | } 282 | } 283 | items = items[start:end] 284 | total = l 285 | } else { 286 | res := r.Store.Query(gvr, store.Query{ 287 | Namespace: namespace, 288 | Paginate: *paginate, 289 | }) 290 | if res.Error != nil { 291 | return errorProxy(r.Writer, v1.Status{ 292 | Status: v1.StatusFailure, 293 | Message: "query error", 294 | Reason: v1.StatusReason(res.Error.Error()), 295 | Code: 400, 296 | }) 297 | } 298 | items = res.Items 299 | total = res.Total 300 | } 301 | apiVersion := "" 302 | if gvr.Group == "" { 303 | apiVersion = gvr.Version 304 | } else { 305 | apiVersion = gvr.Group + "/" + gvr.Version 306 | } 307 | var remainCount int64 308 | if paginate.Page == 0 && paginate.PageSize == 0 { 309 | // all item returned 310 | remainCount = 0 311 | } else { 312 | // page starts with 1, 313 | remainCount = total - (paginate.PageSize * paginate.Page) 314 | if remainCount < 0 && len(items) == 0 && paginate.Page != 1 { 315 | return errorProxy(r.Writer, v1.Status{ 316 | Status: v1.StatusFailure, 317 | Message: "out of page", 318 | Reason: v1.StatusReason(fmt.Sprintf("request resources out of page: %d", remainCount)), 319 | Code: 400, 320 | }) 321 | } else if remainCount < 0 { 322 | remainCount = 0 323 | } 324 | } 325 | if strings.Contains(r.Request.Header.Get("accept"), "application/json;as=Table") { 326 | return serverPrint(items) 327 | } 328 | return map[string]interface{}{ 329 | "apiVersion": apiVersion, 330 | "kind": common.GetGVRKind(gvr.Group, gvr.Version, gvr.Resource), 331 | "metadata": map[string]interface{}{ 332 | "selfLink": r.Request.URL.Path, 333 | "remainingItemCount": remainCount, 334 | }, 335 | "items": items, 336 | } 337 | } 338 | 339 | func serverPrint(items []interface{}) interface{} { 340 | table := v1.Table{ 341 | TypeMeta: v1.TypeMeta{ 342 | Kind: "Table", 343 | APIVersion: "meta.k8s.io/v1", 344 | }, 345 | } 346 | indexMap := map[string]int{} 347 | for i, item := range items { 348 | if oo, ok := item.(v1.Object); ok { 349 | indexesStr := oo.GetAnnotations()[constants.IndexAnno] 350 | if len(indexesStr) == 0 { 351 | continue 352 | } 353 | indexes := map[string]string{} 354 | _ = json.Unmarshal([]byte(indexesStr), &indexes) 355 | if i == 0 { 356 | commonCols := []string{"cluster", "namespace", "name"} 357 | if _, ok := indexes["namespace"]; !ok { 358 | commonCols = []string{"cluster", "name"} 359 | } 360 | cols := []string{} 361 | for k := range indexes { 362 | switch k { 363 | case "cluster": 364 | case "namespace": 365 | case "name": 366 | continue 367 | default: 368 | cols = append(cols, k) 369 | } 370 | } 371 | sort.Slice(cols, func(i, j int) bool { 372 | return cols[i] < cols[j] 373 | }) 374 | cols = append(commonCols, cols...) 375 | for j, c := range cols { 376 | table.ColumnDefinitions = append(table.ColumnDefinitions, v1.TableColumnDefinition{ 377 | Name: c, 378 | Type: "string", 379 | Priority: func() int32 { 380 | switch c { 381 | case "cluster": 382 | return 1 383 | case "is_deleted", "labels", "created_at": 384 | return 2 385 | } 386 | return 0 387 | }(), 388 | }) 389 | indexMap[c] = j 390 | } 391 | } 392 | cells := make([]interface{}, len(indexMap)) 393 | for k, v := range indexes { 394 | cells[indexMap[k]] = v 395 | } 396 | raw := v1.TableRow{ 397 | Cells: cells, 398 | } 399 | table.Rows = append(table.Rows, raw) 400 | } 401 | } 402 | return table 403 | } 404 | 405 | func isWatchRequest(r *http.Request) bool { 406 | query := r.URL.Query() 407 | if w, ok := query["watch"]; ok { 408 | ws := strings.ToLower(w[0]) 409 | if ws == "1" || ws == "y" || ws == "true" { 410 | return true 411 | } 412 | } 413 | if strings.Contains(r.URL.Path, "/watch/") { 414 | return true 415 | } 416 | return false 417 | } 418 | 419 | func proxyPassWatch(r *ReqContext, cluster string) interface{} { 420 | q := r.Request.URL.Query() 421 | q.Set("timeout", "30m") 422 | if v, ok := q["labelSelector"]; ok { 423 | if len(v) == 1 && v[0] == "" { 424 | delete(q, "labelSelector") 425 | } 426 | } 427 | r.Request.URL.RawQuery = q.Encode() 428 | u := r.Request.URL.String() 429 | log.Debugf("proxyPass url: %s", u) 430 | timeout := 30 * time.Minute 431 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 432 | defer cancel() 433 | reader, err := getRequest(r, cluster, timeout).RequestURI(u).Stream(ctx) 434 | if err != nil { 435 | if es, ok := err.(*errors.StatusError); ok { 436 | return errorProxy(r.Writer, es.ErrStatus) 437 | } 438 | return err 439 | } 440 | r.Writer.Header().Set("Content-Type", "application/json") 441 | r.Writer.Header().Set("Transfer-Encoding", "chunked") 442 | r.Writer.Header().Set("Connection", "keep-alive") 443 | buf := bytes.NewBuffer([]byte{}) 444 | for { 445 | t := make([]byte, 1) 446 | _, err := reader.Read(t) 447 | if err != nil { 448 | _, _ = r.Writer.Write(buf.Bytes()) 449 | return nil 450 | } 451 | buf.Write(t) 452 | if t[0] == '\n' { 453 | _, _ = r.Writer.Write(buf.Bytes()) 454 | buf.Reset() 455 | } 456 | select { 457 | case <-r.Request.Context().Done(): 458 | return nil 459 | default: 460 | } 461 | } 462 | } 463 | 464 | func getRequest(r *ReqContext, cluster string, timeout time.Duration) *rest.Request { 465 | c := r.ClusterClients[cluster].Discovery().RESTClient().(*rest.RESTClient) 466 | c.Client.Timeout = timeout 467 | var req *rest.Request 468 | switch r.Request.Method { 469 | case http.MethodGet: 470 | return c.Get() 471 | case http.MethodPost: 472 | req = c.Post() 473 | case http.MethodDelete: 474 | req = c.Delete() 475 | case http.MethodPut: 476 | req = c.Put() 477 | case http.MethodPatch: 478 | req = c.Patch(types.PatchType(r.Request.Header.Get("Content-Type"))) 479 | default: 480 | log.Errorf("unexpected method: %s", r.Request.Method) 481 | return nil 482 | } 483 | // for k, v := range r.Request.Header { 484 | // if len(v) > 0 { 485 | // req = req.SetHeader(k, v[0]) 486 | // } 487 | // } 488 | req = req.Body(r.Request.Body) 489 | return req 490 | } 491 | 492 | func proxyPass(r *ReqContext, cluster string) interface{} { 493 | if cluster == "" { 494 | cluster = common.GetConfig().DefaultCluster 495 | } 496 | if _, ok := r.ClusterClients[cluster]; !ok { 497 | return errorProxy(r.Writer, v1.Status{ 498 | Status: v1.StatusFailure, 499 | Message: "cluster not found", 500 | Reason: v1.StatusReason(fmt.Sprintf("request cluster not found: %s", cluster)), 501 | Code: 404, 502 | }) 503 | } 504 | if isWatchRequest(r.Request) { 505 | return proxyPassWatch(r, cluster) 506 | } 507 | u := r.Request.URL.String() 508 | log.Debugf("proxyPass url: %s", u) 509 | timeout := time.Minute 510 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 511 | defer cancel() 512 | res, err := getRequest(r, cluster, timeout).RequestURI(u).DoRaw(ctx) 513 | if err != nil { 514 | if es, ok := err.(*errors.StatusError); ok { 515 | return errorProxy(r.Writer, es.ErrStatus) 516 | } 517 | return err 518 | } 519 | r.Writer.Header().Set("Content-Type", "application/json") 520 | return res 521 | } 522 | -------------------------------------------------------------------------------- /api/proxy_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | v1 "k8s.io/api/core/v1" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/runtime" 13 | "k8s.io/client-go/kubernetes" 14 | "k8s.io/client-go/kubernetes/fake" 15 | 16 | "github.com/DaoCloud/ckube/common" 17 | "github.com/DaoCloud/ckube/store" 18 | ) 19 | 20 | type fakeWriter struct { 21 | bs []byte 22 | code int 23 | } 24 | 25 | func (f *fakeWriter) Header() http.Header { 26 | return http.Header{} 27 | } 28 | 29 | func (f *fakeWriter) Write(bytes []byte) (int, error) { 30 | f.bs = bytes 31 | return len(bytes), nil 32 | } 33 | 34 | func (f *fakeWriter) WriteHeader(statusCode int) { 35 | f.code = statusCode 36 | } 37 | 38 | type fakeStore struct { 39 | store.Store 40 | storeResources store.QueryResult 41 | } 42 | 43 | func (f fakeStore) Query(gvr store.GroupVersionResource, query store.Query) store.QueryResult { 44 | return f.storeResources 45 | } 46 | 47 | func (f fakeStore) IsStoreGVR(gvr store.GroupVersionResource) bool { 48 | return gvr.Group == "" && gvr.Version == "v1" && gvr.Resource == "pods" 49 | } 50 | 51 | type fakeValueContext struct { 52 | context.Context 53 | resultMap map[string]string 54 | } 55 | 56 | func (c fakeValueContext) Value(key interface{}) interface{} { 57 | return c.resultMap 58 | } 59 | 60 | var podsMap = map[string]string{ 61 | "namespace": "default", 62 | "group": "", 63 | "version": "v1", 64 | "resourceType": "pods", 65 | } 66 | 67 | func podsInterfaces(pods []v1.Pod) []interface{} { 68 | a := []interface{}{} 69 | for _, p := range pods { 70 | a = append(a, p.DeepCopy()) 71 | } 72 | return a 73 | } 74 | 75 | var testPods = podsInterfaces([]v1.Pod{ 76 | { 77 | ObjectMeta: metav1.ObjectMeta{ 78 | Name: "test", 79 | }, 80 | }, 81 | }) 82 | 83 | func TestProxy(t *testing.T) { 84 | common.InitConfig(&common.Config{Proxies: []common.Proxy{ 85 | { 86 | Group: "", 87 | Version: "v1", 88 | Resource: "pods", 89 | ListKind: "PodList", 90 | }, 91 | }}) 92 | cases := []struct { 93 | name string 94 | path string 95 | storeResources store.QueryResult 96 | contextMap map[string]string 97 | kubeResources []runtime.Object 98 | expectCode int 99 | expectRes interface{} 100 | }{ 101 | { 102 | name: "query pods", 103 | path: "/api/v1/pods", 104 | contextMap: podsMap, 105 | storeResources: store.QueryResult{ 106 | Items: testPods, 107 | Total: 1, 108 | }, 109 | expectCode: 0, 110 | expectRes: map[string]interface{}( 111 | map[string]interface{}{ 112 | "apiVersion": "v1", 113 | "items": testPods, 114 | "kind": "PodList", 115 | "metadata": map[string]interface{}{"remainingItemCount": int64(0), "selfLink": "/api/v1/pods"}}), 116 | }, 117 | { 118 | name: "query pods with label selector", 119 | path: "/api/v1/pods?labelSelector=test=1", 120 | contextMap: podsMap, 121 | storeResources: store.QueryResult{ 122 | Items: podsInterfaces([]v1.Pod{ 123 | { 124 | ObjectMeta: metav1.ObjectMeta{ 125 | Name: "test", 126 | Labels: map[string]string{ 127 | "test": "1", 128 | }, 129 | }, 130 | }, 131 | { 132 | ObjectMeta: metav1.ObjectMeta{ 133 | Name: "test1", 134 | Labels: map[string]string{ 135 | "test": "2", 136 | }, 137 | }, 138 | }, 139 | }), 140 | Total: 2, 141 | }, 142 | expectCode: 0, 143 | expectRes: map[string]interface{}( 144 | map[string]interface{}{ 145 | "apiVersion": "v1", 146 | "items": podsInterfaces([]v1.Pod{ 147 | { 148 | ObjectMeta: metav1.ObjectMeta{ 149 | Name: "test", 150 | Labels: map[string]string{ 151 | "test": "1", 152 | }, 153 | }, 154 | }, 155 | }), 156 | "kind": "PodList", 157 | "metadata": map[string]interface{}{"remainingItemCount": int64(0), "selfLink": "/api/v1/pods"}}), 158 | }, 159 | } 160 | for i, c := range cases { 161 | t.Run(fmt.Sprintf("%d---%s", i, c.name), func(t *testing.T) { 162 | req, _ := http.NewRequestWithContext( 163 | fakeValueContext{resultMap: c.contextMap}, 164 | "GET", 165 | c.path, 166 | nil, 167 | ) 168 | s := fakeStore{ 169 | storeResources: c.storeResources, 170 | } 171 | client := fake.NewSimpleClientset(c.kubeResources...) 172 | writer := fakeWriter{} 173 | res := Proxy(&ReqContext{ 174 | ClusterClients: map[string]kubernetes.Interface{"": client}, 175 | Store: s, 176 | Request: req, 177 | Writer: &writer, 178 | }) 179 | assert.Equal(t, c.expectCode, writer.code) 180 | assert.Equal(t, c.expectRes, res) 181 | }) 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /api/req.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/DaoCloud/ckube/store" 7 | "k8s.io/client-go/kubernetes" 8 | ) 9 | 10 | type ReqContext struct { 11 | ClusterClients map[string]kubernetes.Interface 12 | Store store.Store 13 | Request *http.Request 14 | Writer http.ResponseWriter 15 | } 16 | -------------------------------------------------------------------------------- /cmd/cacheproxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | "k8s.io/client-go/kubernetes" 11 | "k8s.io/client-go/rest" 12 | "k8s.io/client-go/tools/clientcmd" 13 | kubeapi "k8s.io/client-go/tools/clientcmd/api/v1" 14 | "sigs.k8s.io/yaml" 15 | 16 | "github.com/DaoCloud/ckube/common" 17 | "github.com/DaoCloud/ckube/log" 18 | "github.com/DaoCloud/ckube/server" 19 | "github.com/DaoCloud/ckube/store" 20 | "github.com/DaoCloud/ckube/store/memory" 21 | "github.com/DaoCloud/ckube/utils" 22 | "github.com/DaoCloud/ckube/utils/prommonitor" 23 | "github.com/DaoCloud/ckube/watcher" 24 | ) 25 | 26 | func GetK8sConfigConfigWithFile(kubeconfig, context string) *rest.Config { 27 | var config *rest.Config 28 | if kubeconfig == "" && context == "" { 29 | config, _ := rest.InClusterConfig() 30 | if config != nil { 31 | return config 32 | } 33 | } 34 | if kubeconfig != "" { 35 | info, err := os.Stat(kubeconfig) 36 | if err != nil || info.Size() == 0 { 37 | // If the specified kubeconfig doesn't exists / empty file / any other error 38 | // from file stat, fall back to default 39 | kubeconfig = "" 40 | } 41 | } 42 | 43 | // Config loading rules: 44 | // 1. kubeconfig if it not empty string 45 | // 2. In cluster config if running in-cluster 46 | // 3. Config(s) in KUBECONFIG environment variable 47 | // 4. Use $HOME/.kube/config 48 | loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() 49 | loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig 50 | loadingRules.ExplicitPath = kubeconfig 51 | configOverrides := &clientcmd.ConfigOverrides{ 52 | ClusterDefaults: clientcmd.ClusterDefaults, 53 | CurrentContext: context, 54 | } 55 | 56 | config, _ = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides).ClientConfig() 57 | return config 58 | } 59 | 60 | func GetKubernetesClientWithFile(kubeconfig, context string) (kubernetes.Interface, error) { 61 | clientset, err := kubernetes.NewForConfig(GetK8sConfigConfigWithFile(kubeconfig, context)) 62 | if err != nil { 63 | return nil, err 64 | } 65 | return clientset, err 66 | } 67 | 68 | func loadFromConfig(kubeConfig, configFile string) (map[string]kubernetes.Interface, watcher.Watcher, store.Store, error) { 69 | 70 | cfg := common.Config{} 71 | if bs, err := os.ReadFile(configFile); err != nil { 72 | log.Errorf("config file load error: %v", err) 73 | return nil, nil, nil, err 74 | } else { 75 | err := json.Unmarshal(bs, &cfg) 76 | if err != nil { 77 | log.Errorf("config file load error: %v", err) 78 | return nil, nil, nil, err 79 | } 80 | } 81 | clusterConfigs := map[string]rest.Config{} 82 | clusterClients := map[string]kubernetes.Interface{} 83 | kubecfg := kubeapi.Config{} 84 | if kubeConfig == "" { 85 | defaultConfig := path.Join(os.Getenv("HOME"), ".kube/config") 86 | if _, err := os.Stat(defaultConfig); err != nil { 87 | // may running in pod 88 | log.Info("no kube config found, load from service account.") 89 | c := GetK8sConfigConfigWithFile(kubeConfig, "") 90 | if c == nil { 91 | log.Errorf("init k8s config from service account error") 92 | return nil, nil, nil, fmt.Errorf("init k8s config error") 93 | } 94 | if cfg.DefaultCluster == "" { 95 | cfg.DefaultCluster = "default" 96 | } 97 | clusterConfigs[cfg.DefaultCluster] = *c 98 | client, err := GetKubernetesClientWithFile(kubeConfig, "") 99 | if err != nil { 100 | return nil, nil, nil, err 101 | } 102 | clusterClients[cfg.DefaultCluster] = client 103 | } else { 104 | kubeConfig = defaultConfig 105 | } 106 | } 107 | if kubeConfig != "" { 108 | bs, err := os.ReadFile(kubeConfig) 109 | if err != nil { 110 | log.Errorf("read kube config error: %v", err) 111 | return nil, nil, nil, err 112 | } 113 | err = yaml.Unmarshal(bs, &kubecfg) 114 | if err != nil { 115 | err = json.Unmarshal(bs, &kubecfg) 116 | if err != nil { 117 | log.Errorf("parse kube config %s error: %v", kubeConfig, err) 118 | return nil, nil, nil, err 119 | } 120 | } 121 | log.Debugf("got kube config: %s", bs) 122 | cfg.DefaultCluster = kubecfg.CurrentContext 123 | 124 | for _, ctx := range kubecfg.Contexts { 125 | c := GetK8sConfigConfigWithFile(kubeConfig, ctx.Name) 126 | if c == nil { 127 | log.Errorf("init k8s config error") 128 | return nil, nil, nil, fmt.Errorf("init k8s config error") 129 | } 130 | clusterConfigs[ctx.Name] = *c 131 | client, err := GetKubernetesClientWithFile(kubeConfig, ctx.Name) 132 | if err != nil { 133 | log.Errorf("init k8s client error: %v", err) 134 | return nil, nil, nil, err 135 | } 136 | clusterClients[ctx.Name] = client 137 | } 138 | } 139 | common.InitConfig(&cfg) 140 | 141 | // 记录组件运行状态 142 | prommonitor.Up.WithLabelValues(prommonitor.CkubeComponent).Set(1) 143 | 144 | indexConf := map[store.GroupVersionResource]map[string]string{} 145 | storeGVRConfig := []store.GroupVersionResource{} 146 | for _, proxy := range cfg.Proxies { 147 | indexConf[store.GroupVersionResource{ 148 | Group: proxy.Group, 149 | Version: proxy.Version, 150 | Resource: proxy.Resource, 151 | }] = proxy.Index 152 | storeGVRConfig = append(storeGVRConfig, store.GroupVersionResource{ 153 | Group: proxy.Group, 154 | Version: proxy.Version, 155 | Resource: proxy.Resource, 156 | }) 157 | } 158 | m := memory.NewMemoryStore(indexConf) 159 | w := watcher.NewWatcher(clusterConfigs, storeGVRConfig, m) 160 | _ = w.Start() 161 | return clusterClients, w, m, nil 162 | } 163 | 164 | func main() { 165 | configFile := "" 166 | listen := ":80" 167 | kubeConfig := "" 168 | debug := false 169 | defaultConfig := path.Join(os.Getenv("HOME"), ".kube/config") 170 | flag.StringVar(&configFile, "c", "config/local.json", "config file path") 171 | flag.StringVar(&listen, "a", ":80", "listen port") 172 | flag.StringVar(&kubeConfig, "k", "", "kube config file name") 173 | flag.BoolVar(&debug, "d", false, "debug mode") 174 | flag.Parse() 175 | if debug { 176 | log.SetDebug() 177 | } 178 | clis, w, s, err := loadFromConfig(kubeConfig, configFile) 179 | if err != nil { 180 | log.Errorf("load from config file error: %v", err) 181 | os.Exit(1) 182 | } 183 | ser := server.NewMuxServer(listen, clis, s) 184 | files := []string{configFile} 185 | if kubeConfig == "" { 186 | files = append(files, defaultConfig) 187 | } else { 188 | files = append(files, kubeConfig) 189 | } 190 | fixedWatcher, err := utils.NewFixedFileWatcher(files) 191 | if err != nil { 192 | log.Errorf("create watcher error: %v", err) 193 | } else { 194 | if err := fixedWatcher.Start(); err != nil { 195 | panic(fmt.Errorf("watcher start error: %v", err)) 196 | } 197 | defer fixedWatcher.Close() 198 | go func() { 199 | for e := range fixedWatcher.Events() { 200 | log.Infof("get file watcher event: %v", e) 201 | switch e.Type { 202 | case utils.EventTypeChanged: 203 | // do reload 204 | case utils.EventTypeError: 205 | log.Errorf("got file watcher error type: file: %s", e.Name) 206 | // do reload 207 | } 208 | clis, rw, rs, err := loadFromConfig(kubeConfig, configFile) 209 | if err != nil { 210 | prommonitor.ConfigReload.WithLabelValues("failed").Inc() 211 | log.Errorf("watcher: reload config error: %v", err) 212 | continue 213 | } 214 | prommonitor.Resources.Reset() 215 | _ = w.Stop() 216 | w = rw 217 | ser.ResetStore(rs, clis) // reset store 218 | prommonitor.ConfigReload.WithLabelValues("success").Inc() 219 | log.Infof("auto reloaded config successfully") 220 | } 221 | }() 222 | } 223 | _ = ser.Run() 224 | } 225 | -------------------------------------------------------------------------------- /cmd/ckube-plugin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "strings" 7 | 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | 10 | "github.com/DaoCloud/ckube/log" 11 | "github.com/DaoCloud/ckube/page" 12 | ) 13 | 14 | const ( 15 | get = iota 16 | create 17 | update 18 | notSupport 19 | ) 20 | 21 | func main() { 22 | clusters := "" 23 | args := []string{} 24 | typ := notSupport 25 | selectorPos := 0 26 | for i := 1; i < len(os.Args); i++ { 27 | a := os.Args[i] 28 | switch a { 29 | case "get": 30 | typ = get 31 | case "create": 32 | typ = create 33 | // case "delete": 34 | // typ = del 35 | } 36 | if a == "--clusters" { 37 | if i+1 < len(os.Args) { 38 | clusters = os.Args[i+1] 39 | i++ 40 | continue 41 | } 42 | } else { 43 | args = append(args, a) 44 | } 45 | if a == "-l" || a == "--selector" { 46 | if i+1 < len(os.Args) { 47 | selectorPos = i + 1 48 | } 49 | } 50 | } 51 | if typ == notSupport { 52 | log.Errorf("ckube plugin not support current subcommad") 53 | os.Exit(1) 54 | } 55 | if clusters == "" { 56 | log.Errorf("you are not specified the --clusters option, we will use the default cluster to process you request") 57 | } else { 58 | cs := strings.Split(clusters, ",") 59 | switch typ { 60 | case get: 61 | p := page.Paginate{} 62 | _ = p.Clusters(cs) 63 | selector := "" 64 | if selectorPos != 0 { 65 | selector = args[selectorPos] 66 | } 67 | o, _ := page.QueryListOptions(metav1.ListOptions{ 68 | LabelSelector: selector, 69 | }, p) 70 | if selectorPos == 0 { 71 | args = append(args, "-l", o.LabelSelector) 72 | } else { 73 | args[selectorPos] = o.LabelSelector 74 | } 75 | case create, update: 76 | if len(cs) > 1 { 77 | log.Errorf("create resource can only specified one cluster, error: %v", clusters) 78 | os.Exit(2) 79 | } 80 | p := page.Paginate{} 81 | _ = p.Clusters(cs) 82 | o, _ := page.QueryCreateOptions(metav1.CreateOptions{}, cs[0]) 83 | args = append(args, "--field-manager", o.FieldManager) 84 | } 85 | } 86 | c := exec.Command("kubectl", args...) 87 | // fmt.Printf("args %v\n", args) 88 | c.Stdin = os.Stdin 89 | c.Stdout = os.Stdout 90 | c.Stderr = os.Stderr 91 | _ = c.Run() 92 | } 93 | -------------------------------------------------------------------------------- /cmd/ckubecli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | 11 | "github.com/DaoCloud/ckube/page" 12 | ) 13 | 14 | func main() { 15 | var page_ int 16 | var pageSize int 17 | var sort string 18 | var search string 19 | var clusters string 20 | flag.IntVar(&page_, "p", 0, "page of result") 21 | flag.IntVar(&pageSize, "s", 0, "page size of result") 22 | flag.StringVar(&sort, "sort", "", "sort of result") 23 | flag.StringVar(&search, "search", "", "search of result") 24 | flag.StringVar(&clusters, "c", "", "clusters of result, comma splited") 25 | flag.Parse() 26 | p := page.Paginate{ 27 | Page: int64(page_), 28 | PageSize: int64(pageSize), 29 | Sort: sort, 30 | Search: search, 31 | } 32 | if clusters != "" { 33 | ccs := strings.Split(clusters, ",") 34 | _ = p.Clusters(ccs) 35 | } 36 | o, err := page.QueryListOptions(v1.ListOptions{}, p) 37 | if err != nil { 38 | fmt.Fprintf(os.Stderr, "query error: %v", err) 39 | os.Exit(1) 40 | } 41 | fmt.Printf(o.LabelSelector) 42 | } 43 | -------------------------------------------------------------------------------- /common/config.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type Proxy struct { 4 | Group string `json:"group"` 5 | Version string `json:"version"` 6 | Resource string `json:"resource"` 7 | ListKind string `json:"list_kind"` 8 | Index map[string]string `json:"index"` 9 | } 10 | 11 | //type Cluster struct { 12 | // Context string `json:"context"` 13 | //} 14 | 15 | type Config struct { 16 | Proxies []Proxy `json:"proxies"` 17 | //Clusters map[string]Cluster `json:"clusters"` 18 | DefaultCluster string `json:"default_cluster"` 19 | Token string `json:"token"` 20 | } 21 | 22 | var cfg *Config 23 | 24 | func InitConfig(c *Config) { 25 | if c.DefaultCluster == "" { 26 | c.DefaultCluster = "default" 27 | } 28 | cfg = c 29 | } 30 | 31 | func GetConfig() Config { 32 | return *cfg 33 | } 34 | 35 | func GetGVRKind(g, v, r string) string { 36 | for _, p := range cfg.Proxies { 37 | if p.Group == g && p.Version == v && p.Resource == r { 38 | return p.ListKind 39 | } 40 | } 41 | return "" 42 | } 43 | -------------------------------------------------------------------------------- /common/constants/ckube.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | PaginateKey = "ckube.daocloud.io/query" 5 | AdvancedSearchPrefix = "__ckube_as__:" 6 | SortASC = "asc" 7 | SortDesc = "desc" 8 | KeyTypeSep = "!" 9 | KeyTypeInt = "int" 10 | KeyTypeStr = "str" 11 | SearchPartsSep = ';' 12 | DSMClusterAnno = "ckube.doacloud.io/cluster" 13 | ClusterPrefix = "dsm-cluster-" 14 | IndexAnno = "ckube.daocloud.io/indexes" 15 | ) 16 | 17 | var ( 18 | _ = PaginateKey 19 | _ = AdvancedSearchPrefix 20 | _ = SortASC 21 | _ = SortDesc 22 | _ = KeyTypeSep 23 | _ = KeyTypeInt 24 | _ = KeyTypeStr 25 | _ = SearchPartsSep 26 | _ = DSMClusterAnno 27 | _ = ClusterPrefix 28 | _ = IndexAnno 29 | ) 30 | -------------------------------------------------------------------------------- /config/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusters": { 3 | "cluster1": { 4 | "context": "dce-admin" 5 | }, 6 | "cluster2": { 7 | "context": "kind-cluster1" 8 | } 9 | }, 10 | "default_cluster": "default", 11 | "token": "", 12 | "proxies": [ 13 | { 14 | "group": "", 15 | "version": "v1", 16 | "resource": "pods", 17 | "list_kind": "PodList", 18 | "index": { 19 | "namespace": "{.metadata.namespace}", 20 | "name": "{.metadata.name}", 21 | "labels": "{.metadata.labels}", 22 | "created_at": "{.metadata.creationTimestamp}" 23 | } 24 | }, 25 | { 26 | "group": "", 27 | "version": "v1", 28 | "resource": "services", 29 | "list_kind": "ServiceList", 30 | "index": { 31 | "namespace": "{.metadata.namespace}", 32 | "name": "{.metadata.name}", 33 | "labels": "{.metadata.labels}", 34 | "created_at": "{.metadata.creationTimestamp}" 35 | } 36 | }, 37 | { 38 | "group": "", 39 | "version": "v1", 40 | "resource": "namespaces", 41 | "list_kind": "NamespaceList", 42 | "index": { 43 | "namespace": "{.metadata.namespace}", 44 | "name": "{.metadata.name}", 45 | "labels": "{.metadata.labels}", 46 | "created_at": "{.metadata.creationTimestamp}" 47 | } 48 | }, 49 | { 50 | "group": "apps", 51 | "version": "v1", 52 | "resource": "deployments", 53 | "list_kind": "DeploymentList", 54 | "index": { 55 | "namespace": "{.metadata.namespace}", 56 | "name": "{.metadata.name}", 57 | "labels": "{.metadata.labels}", 58 | "created_at": "{.metadata.creationTimestamp}" 59 | } 60 | }, 61 | { 62 | "group": "apps", 63 | "version": "v1", 64 | "resource": "replicasets", 65 | "list_kind": "ReplicaSetList", 66 | "index": { 67 | "namespace": "{.metadata.namespace}", 68 | "name": "{.metadata.name}", 69 | "labels": "{.metadata.labels}", 70 | "created_at": "{.metadata.creationTimestamp}" 71 | } 72 | }, 73 | { 74 | "group": "networking.istio.io", 75 | "version": "v1alpha3", 76 | "resource": "destinationrules", 77 | "list_kind": "DestinationRuleList", 78 | "index": { 79 | "namespace": "{.metadata.namespace}", 80 | "name": "{.metadata.name}", 81 | "labels": "{.metadata.labels}", 82 | "created_at": "{.metadata.creationTimestamp}" 83 | } 84 | }, 85 | { 86 | "group": "networking.istio.io", 87 | "version": "v1alpha3", 88 | "resource": "virtualservices", 89 | "list_kind": "VirtualServiceList", 90 | "index": { 91 | "namespace": "{.metadata.namespace}", 92 | "name": "{.metadata.name}", 93 | "labels": "{.metadata.labels}", 94 | "created_at": "{.metadata.creationTimestamp}" 95 | } 96 | } 97 | ] 98 | } 99 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | token=$(cat /run/secrets/kubernetes.io/serviceaccount/token) 6 | server=${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} 7 | 8 | sed -i "s/__KUBE_TOKEN__/${token}/g" /etc/nginx/nginx.conf 9 | sed -i "s/__KUBE_SERVER__/${server}/g" /etc/nginx/nginx.conf 10 | 11 | # start nginx 12 | nginx 13 | 14 | /app/dist/cacheproxy $@ 15 | -------------------------------------------------------------------------------- /examples/paginate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/DaoCloud/ckube/page" 7 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/client-go/kubernetes" 9 | "k8s.io/client-go/rest" 10 | ) 11 | 12 | func main() { 13 | client := kubernetes.NewForConfigOrDie(&rest.Config{ 14 | Host: "http://127.0.0.1:3033", 15 | }) 16 | p := page.Paginate{ 17 | Page: 2, 18 | PageSize: 50, 19 | } 20 | op, _ := page.QueryListOptions(v1.ListOptions{}, p) 21 | podList, err := client.CoreV1().Pods("").List( 22 | context.Background(), 23 | op, 24 | ) 25 | if err != nil { 26 | panic(err) 27 | } 28 | p = page.MakeupResPaginate(podList, p) 29 | fmt.Printf("total of pods: %d, got %d pods", p.Total, len(podList.Items)) 30 | } 31 | -------------------------------------------------------------------------------- /examples/search/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/DaoCloud/ckube/page" 7 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/client-go/kubernetes" 9 | "k8s.io/client-go/rest" 10 | ) 11 | 12 | func main() { 13 | client := kubernetes.NewForConfigOrDie(&rest.Config{ 14 | Host: "http://127.0.0.1:3033", 15 | }) 16 | p := page.Paginate{ 17 | // full search 18 | Search: `name="default"`, 19 | } 20 | op, _ := page.QueryListOptions(v1.ListOptions{}, p) 21 | podList, err := client.CoreV1().Namespaces().List( 22 | context.Background(), 23 | op, 24 | ) 25 | if err != nil { 26 | panic(err) 27 | } 28 | p = page.MakeupResPaginate(podList, p) 29 | fmt.Printf("total of default namespaces: %d, got %d\n", p.Total, len(podList.Items)) 30 | p = page.Paginate{ 31 | Page: 1, 32 | PageSize: 5, 33 | Search: `e`, 34 | } 35 | op, _ = page.QueryListOptions(v1.ListOptions{}, p) 36 | podList, err = client.CoreV1().Namespaces().List( 37 | context.Background(), 38 | op, 39 | ) 40 | if err != nil { 41 | panic(err) 42 | } 43 | p = page.MakeupResPaginate(podList, p) 44 | fmt.Printf("total of namespaces which containes e: %d, got %d\n", p.Total, len(podList.Items)) 45 | } 46 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/DaoCloud/ckube 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/fsnotify/fsnotify v1.4.9 7 | github.com/gorilla/mux v1.8.0 8 | github.com/prometheus/client_golang v1.7.1 9 | github.com/samber/lo v1.27.0 10 | github.com/sirupsen/logrus v1.8.1 11 | github.com/stretchr/testify v1.7.1 12 | k8s.io/api v0.21.0 13 | k8s.io/apimachinery v0.21.0 14 | k8s.io/client-go v0.21.0 15 | sigs.k8s.io/yaml v1.2.0 16 | ) 17 | 18 | require ( 19 | github.com/beorn7/perks v1.0.1 // indirect 20 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/evanphx/json-patch v4.9.0+incompatible // indirect 23 | github.com/go-logr/logr v0.4.0 // indirect 24 | github.com/gogo/protobuf v1.3.2 // indirect 25 | github.com/golang/protobuf v1.4.3 // indirect 26 | github.com/google/go-cmp v0.5.2 // indirect 27 | github.com/google/gofuzz v1.2.0 // indirect 28 | github.com/googleapis/gnostic v0.5.1 // indirect 29 | github.com/imdario/mergo v0.3.5 // indirect 30 | github.com/json-iterator/go v1.1.10 // indirect 31 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 32 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 33 | github.com/modern-go/reflect2 v1.0.1 // indirect 34 | github.com/pkg/errors v0.9.1 // indirect 35 | github.com/pmezard/go-difflib v1.0.0 // indirect 36 | github.com/prometheus/client_model v0.2.0 // indirect 37 | github.com/prometheus/common v0.10.0 // indirect 38 | github.com/prometheus/procfs v0.1.3 // indirect 39 | github.com/spf13/pflag v1.0.5 // indirect 40 | golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect 41 | golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect 42 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect 43 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect 44 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect 45 | golang.org/x/text v0.3.4 // indirect 46 | golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect 47 | google.golang.org/appengine v1.6.6 // indirect 48 | google.golang.org/protobuf v1.25.0 // indirect 49 | gopkg.in/inf.v0 v0.9.1 // indirect 50 | gopkg.in/yaml.v2 v2.4.0 // indirect 51 | gopkg.in/yaml.v3 v3.0.1 // indirect 52 | k8s.io/klog/v2 v2.8.0 // indirect 53 | k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect 54 | k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect 55 | sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect 56 | ) 57 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= 9 | cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= 10 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= 11 | cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= 12 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 13 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= 14 | cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= 15 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 16 | cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= 17 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 18 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= 19 | cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= 20 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 21 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= 22 | cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= 23 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 24 | github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= 25 | github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= 26 | github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= 27 | github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= 28 | github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= 29 | github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= 30 | github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= 31 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 32 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 33 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 34 | github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 35 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 36 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 37 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 38 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 39 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 40 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 41 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 42 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 43 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 44 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 45 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 46 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 47 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 48 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 49 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 50 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 51 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 52 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 53 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 54 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 55 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 56 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= 57 | github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 58 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 59 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 60 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 61 | github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= 62 | github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 63 | github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= 64 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 65 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 66 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 67 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 68 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 69 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 70 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 71 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 72 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 73 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 74 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 75 | github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= 76 | github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= 77 | github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= 78 | github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 79 | github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= 80 | github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= 81 | github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= 82 | github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 83 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 84 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 85 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 86 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 87 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 88 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 89 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 90 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 91 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 92 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 93 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 94 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 95 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 96 | github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 97 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 98 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 99 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 100 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 101 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 102 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 103 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 104 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 105 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 106 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 107 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 108 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 109 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 110 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 111 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 112 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 113 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 114 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 115 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 116 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 117 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 118 | github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= 119 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 120 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 121 | github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 122 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 123 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 124 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 125 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 126 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 127 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 128 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 129 | github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 130 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 131 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 132 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 133 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 134 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 135 | github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= 136 | github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= 137 | github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= 138 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 139 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 140 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 141 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 142 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 143 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 144 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 145 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 146 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 147 | github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= 148 | github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 149 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 150 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 151 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 152 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 153 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 154 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 155 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 156 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 157 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 158 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 159 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 160 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 161 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 162 | github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= 163 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 164 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 165 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 166 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 167 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 168 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 169 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 170 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 171 | github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= 172 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 173 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 174 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 175 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 176 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 177 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 178 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 179 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 180 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 181 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 182 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 183 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 184 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 185 | github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= 186 | github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 187 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 188 | github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= 189 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 190 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 191 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 192 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 193 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 194 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 195 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 196 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 197 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 198 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 199 | github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= 200 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 201 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 202 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 203 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 204 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 205 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 206 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 207 | github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= 208 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 209 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 210 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 211 | github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= 212 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 213 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 214 | github.com/samber/lo v1.27.0 h1:GOyDWxsblvqYobqsmUuMddPa2/mMzkKyojlXol4+LaQ= 215 | github.com/samber/lo v1.27.0/go.mod h1:it33p9UtPMS7z72fP4gw/EIfQB2eI8ke7GR2wc6+Rhg= 216 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 217 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 218 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= 219 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 220 | github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= 221 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 222 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 223 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 224 | github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= 225 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 226 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 227 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 228 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 229 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 230 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 231 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 232 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 233 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 234 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 235 | github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= 236 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 237 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 238 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 239 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 240 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 241 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 242 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 243 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 244 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 245 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 246 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 247 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 248 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 249 | golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 250 | golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 251 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 252 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 253 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 254 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 255 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 256 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 257 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 258 | golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 259 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= 260 | golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= 261 | golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= 262 | golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= 263 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 264 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 265 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 266 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 267 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 268 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 269 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 270 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 271 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 272 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 273 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 274 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 275 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 276 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 277 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 278 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 279 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 280 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 281 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 282 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 283 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 284 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 285 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 286 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 287 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 288 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 289 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 290 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 291 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 292 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 293 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 294 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 295 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 296 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 297 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 298 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 299 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 300 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 301 | golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 302 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 303 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 304 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 305 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 306 | golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= 307 | golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 308 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 309 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 310 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 311 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 312 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= 313 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 314 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 315 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 316 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 317 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 318 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 319 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 320 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 321 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 322 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 323 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 324 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 325 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 326 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 327 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 328 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 329 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 330 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 331 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 332 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 333 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 334 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 335 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 336 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 337 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 338 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 339 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 340 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 341 | golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 342 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 343 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 344 | golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 345 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 346 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 347 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 348 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 349 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 350 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 351 | golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 352 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= 353 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 354 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 355 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 356 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= 357 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 358 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 359 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 360 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 361 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 362 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 363 | golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= 364 | golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 365 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 366 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 367 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 368 | golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= 369 | golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 370 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 371 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 372 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 373 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 374 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 375 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 376 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 377 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 378 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 379 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 380 | golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 381 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 382 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 383 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 384 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 385 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 386 | golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 387 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 388 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 389 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 390 | golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 391 | golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 392 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 393 | golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 394 | golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 395 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 396 | golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 397 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 398 | golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 399 | golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 400 | golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 401 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 402 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 403 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 404 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 405 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 406 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 407 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 408 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 409 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 410 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 411 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 412 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 413 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 414 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 415 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 416 | google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 417 | google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 418 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 419 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 420 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 421 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 422 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 423 | google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= 424 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 425 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 426 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 427 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 428 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 429 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 430 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 431 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 432 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 433 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 434 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 435 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 436 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 437 | google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 438 | google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 439 | google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= 440 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 441 | google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 442 | google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 443 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 444 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 445 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 446 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 447 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 448 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 449 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 450 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 451 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 452 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 453 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 454 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 455 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 456 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 457 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 458 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 459 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 460 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 461 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 462 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 463 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 464 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 465 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 466 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 467 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 468 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 469 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 470 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 471 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 472 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 473 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 474 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 475 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 476 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 477 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 478 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 479 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 480 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 481 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 482 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 483 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 484 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 485 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 486 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 487 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 488 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 489 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 490 | honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 491 | k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= 492 | k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= 493 | k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= 494 | k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= 495 | k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= 496 | k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= 497 | k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 498 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 499 | k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= 500 | k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= 501 | k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= 502 | k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= 503 | k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= 504 | k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= 505 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 506 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= 507 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 508 | sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= 509 | sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= 510 | sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= 511 | sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= 512 | sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= 513 | -------------------------------------------------------------------------------- /kube/kube.go: -------------------------------------------------------------------------------- 1 | package kube 2 | 3 | import ( 4 | "os" 5 | 6 | "k8s.io/client-go/rest" 7 | "k8s.io/client-go/tools/clientcmd" 8 | ) 9 | 10 | func GetK8sConfigConfigWithFile(kubeconfig, context string) (*rest.Config, error) { 11 | config, _ := rest.InClusterConfig() 12 | if config != nil { 13 | return config, nil 14 | } 15 | if kubeconfig != "" { 16 | info, err := os.Stat(kubeconfig) 17 | if err != nil || info.Size() == 0 { 18 | // If the specified kubeconfig doesn't exists / empty file / any other error 19 | // from file stat, fall back to default 20 | kubeconfig = "" 21 | } 22 | } 23 | 24 | // Config loading rules: 25 | // 1. kubeconfig if it not empty string 26 | // 2. In cluster config if running in-cluster 27 | // 3. Config(s) in KUBECONFIG environment variable 28 | // 4. Use $HOME/.kube/config 29 | loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() 30 | loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig 31 | loadingRules.ExplicitPath = kubeconfig 32 | configOverrides := &clientcmd.ConfigOverrides{ 33 | ClusterDefaults: clientcmd.ClusterDefaults, 34 | CurrentContext: context, 35 | } 36 | 37 | config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides).ClientConfig() 38 | 39 | return config, err 40 | } 41 | -------------------------------------------------------------------------------- /kube/label.go: -------------------------------------------------------------------------------- 1 | package kube 2 | 3 | import ( 4 | "fmt" 5 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | k8labels "k8s.io/apimachinery/pkg/labels" 7 | "k8s.io/apimachinery/pkg/selection" 8 | "sort" 9 | "strings" 10 | ) 11 | 12 | const eachPartLen = 56 13 | 14 | func SplittingValue(value string) []string { 15 | res := []string{} 16 | if len(value) <= eachPartLen { 17 | return []string{value} 18 | } 19 | for i := 0; ; i += eachPartLen { 20 | end := i + eachPartLen 21 | if len(value) <= end { 22 | res = append(res, fmt.Sprintf("%04d.%s", i, value[i:])) 23 | break 24 | } 25 | res = append(res, fmt.Sprintf("%04d.%s", i, value[i:end])) 26 | } 27 | return res 28 | } 29 | 30 | func MergeValues(values []string) (string, error) { 31 | if len(values) == 1 { 32 | return values[0], nil 33 | } 34 | sort.Strings(values) 35 | b := strings.Builder{} 36 | for _, v := range values { 37 | parts := strings.Split(v, ".") 38 | if len(parts) != 2 { 39 | return "", fmt.Errorf("value format error") 40 | } 41 | b.WriteString(parts[1]) 42 | } 43 | return b.String(), nil 44 | } 45 | 46 | func ParseToLabelSelector(selector string) (*v1.LabelSelector, error) { 47 | reqs, err := k8labels.ParseToRequirements(selector) 48 | if err != nil { 49 | return nil, fmt.Errorf("couldn't parse the selector string \"%s\": %v", selector, err) 50 | } 51 | 52 | labelSelector := &v1.LabelSelector{ 53 | MatchLabels: map[string]string{}, 54 | MatchExpressions: []v1.LabelSelectorRequirement{}, 55 | } 56 | for _, req := range reqs { 57 | var op v1.LabelSelectorOperator 58 | switch req.Operator() { 59 | case selection.Equals, selection.DoubleEquals: 60 | vals := req.Values() 61 | if vals.Len() != 1 { 62 | return nil, fmt.Errorf("equals operator must have exactly one value") 63 | } 64 | val, ok := vals.PopAny() 65 | if !ok { 66 | return nil, fmt.Errorf("equals operator has exactly one value but it cannot be retrieved") 67 | } 68 | labelSelector.MatchLabels[req.Key()] = val 69 | continue 70 | case selection.In: 71 | op = v1.LabelSelectorOpIn 72 | case selection.NotIn, selection.NotEquals: 73 | op = v1.LabelSelectorOpNotIn 74 | case selection.Exists: 75 | op = v1.LabelSelectorOpExists 76 | case selection.DoesNotExist: 77 | op = v1.LabelSelectorOpDoesNotExist 78 | case selection.GreaterThan, selection.LessThan: 79 | // Adding a separate case for these operators to indicate that this is deliberate 80 | return nil, fmt.Errorf("%q isn't supported in label selectors", req.Operator()) 81 | default: 82 | return nil, fmt.Errorf("%q is not a valid label selector operator", req.Operator()) 83 | } 84 | labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, v1.LabelSelectorRequirement{ 85 | Key: req.Key(), 86 | Operator: op, 87 | Values: req.Values().List(), 88 | }) 89 | } 90 | return labelSelector, nil 91 | } 92 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "strings" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var ( 13 | Debug = log.Debug 14 | Debugf = log.Debugf 15 | Info = log.Info 16 | Infof = log.Infof 17 | Warn = log.Warn 18 | Warning = log.Warn 19 | Warnf = log.Warnf 20 | Error = log.Error 21 | Errorf = log.Errorf 22 | WithField = log.WithField 23 | AccessLog = log.New() 24 | ) 25 | 26 | func init() { 27 | log.SetFormatter(&log.TextFormatter{ 28 | DisableTimestamp: false, 29 | FullTimestamp: true, 30 | DisableLevelTruncation: true, 31 | DisableColors: true, 32 | CallerPrettyfier: func(f *runtime.Frame) (string, string) { 33 | fs := strings.Split(f.File, "/") 34 | filename := fs[len(fs)-1] 35 | ff := strings.Split(f.Function, "/") 36 | _f := ff[len(ff)-1] 37 | return fmt.Sprintf("%s()", _f), fmt.Sprintf("%s:%d", filename, f.Line) 38 | }, 39 | }) 40 | log.SetOutput(os.Stdout) 41 | log.SetLevel(log.InfoLevel) 42 | log.SetReportCaller(true) 43 | AccessLog.SetFormatter(&log.TextFormatter{ 44 | DisableColors: true, 45 | FullTimestamp: true, 46 | }) 47 | } 48 | 49 | func SetDebug() { 50 | log.SetLevel(log.DebugLevel) 51 | } 52 | 53 | type Config struct { 54 | Output string `json:"output"` // 文件输出路径，不填输出终端 55 | Debug bool `json:"debug"` 56 | } 57 | 58 | func InitEngine(config *Config) { 59 | if config != nil && config.Debug { 60 | SetDebug() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes 1; 3 | error_log /var/log/nginx/error.log warn; 4 | pid /var/run/nginx.pid; 5 | events { 6 | worker_connections 1024; 7 | } 8 | http { 9 | include /etc/nginx/mime.types; 10 | default_type application/octet-stream; 11 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 12 | '$status $body_bytes_sent "$http_referer" ' 13 | '"$http_user_agent" "$http_x_forwarded_for"'; 14 | access_log /var/log/nginx/access.log main; 15 | sendfile on; 16 | 17 | upstream kube { 18 | server __KUBE_SERVER__; 19 | } 20 | 21 | upstream proxy { 22 | server 127.0.0.1:3033; 23 | } 24 | 25 | server { 26 | listen 80; 27 | server_name localhost; 28 | location / { 29 | proxy_http_version 1.1; 30 | proxy_buffering off; 31 | proxy_read_timeout 30m; 32 | proxy_set_header Upgrade $http_upgrade; 33 | proxy_set_header Connection "upgrade"; 34 | proxy_set_header Host $http_host; 35 | proxy_set_header X-Real-IP $remote_addr; 36 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 37 | proxy_set_header 'Authorization' "Bearer __KUBE_TOKEN__"; 38 | 39 | proxy_pass https://kube; 40 | } 41 | location /custom { 42 | proxy_http_version 1.1; 43 | proxy_set_header Upgrade $http_upgrade; 44 | proxy_set_header Connection "upgrade"; 45 | proxy_pass http://proxy; 46 | } 47 | location /metrics { 48 | proxy_http_version 1.1; 49 | proxy_set_header Upgrade $http_upgrade; 50 | proxy_set_header Connection "upgrade"; 51 | proxy_pass http://proxy; 52 | } 53 | location /healthy { 54 | proxy_http_version 1.1; 55 | proxy_set_header Upgrade $http_upgrade; 56 | proxy_set_header Connection "upgrade"; 57 | proxy_pass http://proxy; 58 | } 59 | set $ACT ""; # G ---> Get Method, N ---> No Watch 60 | if ($request_method = GET) { 61 | set $ACT "${ACT}G"; 62 | } 63 | if ($arg_watch !~* (1)|(true)) { 64 | set $ACT "${ACT}N"; 65 | } 66 | set $BACKEND "https://kube"; 67 | if ($ACT = "GN") { 68 | set $BACKEND "http://proxy"; 69 | } 70 | # /api/v1/pods /api/v1/namespaces/default/pods /api/v1/namespaces/default/pods/xxx 71 | # /apis/apps/v1/deployments /apis/apps/v1/namespaces/default/deployments /apis/apps/v1/namespaces/default/deployments/xxx 72 | location ~ (((^/api)|(^/apis/[0-9a-zA-Z\.\-\_]+))/[0-9a-zA-Z\.\-\_]+/(namespaces/[0-9a-zA-Z\.\-\_]+/)?[0-9a-zA-Z\.\-\_]+(/[^/]+)?$) { 73 | proxy_http_version 1.1; 74 | proxy_buffering off; 75 | proxy_read_timeout 30m; 76 | proxy_set_header Upgrade $http_upgrade; 77 | proxy_set_header Connection "upgrade"; 78 | proxy_set_header Host $http_host; 79 | proxy_set_header X-Real-IP $remote_addr; 80 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 81 | proxy_set_header 'Authorization' "Bearer __KUBE_TOKEN__"; 82 | 83 | proxy_pass $BACKEND; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /page/page.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "reflect" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/DaoCloud/ckube/common/constants" 12 | "github.com/DaoCloud/ckube/kube" 13 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/apimachinery/pkg/labels" 15 | ) 16 | 17 | type Paginate struct { 18 | Page int64 `json:"page,omitempty" form:"page"` 19 | PageSize int64 `json:"page_size,omitempty" form:"page_size"` 20 | Total int64 `json:"total,omitempty" form:"total" ` 21 | Sort string `json:"sort,omitempty" form:"sort"` 22 | Search string `json:"search,omitempty" form:"search"` 23 | } 24 | 25 | func (p *Paginate) Match(m map[string]string) (bool, error) { 26 | parts := p.SearchParts() 27 | return Match(m, parts) 28 | } 29 | 30 | func (p *Paginate) SearchParts() []string { 31 | search := p.Search 32 | parts := []string{} 33 | start := 0 34 | l := len(search) 35 | trimPart := func(origin string) string { 36 | return strings.ReplaceAll(origin, ";;", ";") 37 | } 38 | for i := 0; i < l-1; i++ { 39 | if search[i] == constants.SearchPartsSep { 40 | if search[i+1] == constants.SearchPartsSep { 41 | // double ;, skip 42 | i += 1 43 | continue 44 | } else { 45 | // need split 46 | parts = append(parts, trimPart(search[start:i])) 47 | start = i + 1 48 | } 49 | } 50 | } 51 | parts = append(parts, trimPart(search[start:])) 52 | ps := []string{} 53 | for _, p := range parts { 54 | if p != "" { 55 | ps = append(ps, p) 56 | } 57 | } 58 | return ps 59 | } 60 | 61 | func parseValue(v string) (string, bool) { 62 | if strings.HasPrefix(v, "!") { 63 | return v[1:], true 64 | } 65 | return v, false 66 | } 67 | 68 | func Match(m map[string]string, searchParts []string) (bool, error) { 69 | if len(searchParts) != 1 { 70 | matched := 0 71 | for _, part := range searchParts { 72 | r, err := Match(m, []string{part}) 73 | if err != nil { 74 | return false, err 75 | } 76 | if r { 77 | matched += 1 78 | } 79 | } 80 | return matched == len(searchParts), nil 81 | } 82 | search := strings.TrimSpace(searchParts[0]) 83 | if search == "" { 84 | return true, nil 85 | } 86 | if strings.HasPrefix(search, constants.AdvancedSearchPrefix) { 87 | if len(search) == len(constants.AdvancedSearchPrefix) { 88 | return false, fmt.Errorf("search format error") 89 | } 90 | selectorStr := search[len(constants.AdvancedSearchPrefix):] 91 | s, err := kube.ParseToLabelSelector(selectorStr) 92 | if err != nil { 93 | return false, err 94 | } 95 | ss, err := v1.LabelSelectorAsSelector(s) 96 | if err != nil { 97 | return false, err 98 | } 99 | return ss.Matches(labels.Set(m)), nil 100 | } 101 | key := "" 102 | value := "" 103 | indexOfEqual := strings.Index(search, "=") 104 | if indexOfEqual < 0 { 105 | // fuzzy search 106 | value = search 107 | } else { 108 | key = search[:indexOfEqual] 109 | if indexOfEqual < len(search)-1 { 110 | value = search[indexOfEqual+1:] 111 | } 112 | } 113 | value, reverse := parseValue(value) 114 | if key != "" { 115 | if v, ok := m[key]; !ok { 116 | return false, fmt.Errorf("unexpected search key: %s", key) 117 | } else { 118 | vv := strings.Contains(strconv.Quote(v), value) 119 | if reverse { 120 | return !vv, nil 121 | } 122 | return vv, nil 123 | } 124 | } 125 | // fuzzy search 126 | for _, v := range m { 127 | vv := strings.Contains(strconv.Quote(v), value) 128 | if reverse { 129 | if vv { 130 | return false, nil 131 | } 132 | } else { 133 | if vv { 134 | return true, nil 135 | } 136 | } 137 | } 138 | return reverse, nil 139 | } 140 | 141 | func (p *Paginate) SearchSelector() (*v1.LabelSelector, error) { 142 | s := v1.LabelSelector{} 143 | parts := p.SearchParts() 144 | search := "" 145 | for _, part := range parts { 146 | if strings.HasPrefix(part, constants.AdvancedSearchPrefix) { 147 | search = part 148 | break 149 | } 150 | } 151 | if search == "" { 152 | return &s, nil 153 | } 154 | if !strings.HasPrefix(search, constants.AdvancedSearchPrefix) { 155 | return nil, fmt.Errorf("") 156 | } 157 | if len(constants.AdvancedSearchPrefix) == len(search) { 158 | return &s, nil 159 | } 160 | return kube.ParseToLabelSelector(search[len(constants.AdvancedSearchPrefix):]) 161 | } 162 | 163 | func (p *Paginate) SetSearchSelector(selector *v1.LabelSelector) error { 164 | parts := p.SearchParts() 165 | sstr := v1.FormatLabelSelector(selector) 166 | if sstr == "" { 167 | return fmt.Errorf("parse selector %v error", selector) 168 | } 169 | pps := []string{constants.AdvancedSearchPrefix + sstr} 170 | for _, part := range parts { 171 | if !strings.HasPrefix(part, constants.AdvancedSearchPrefix) { 172 | pps = append(pps, part) 173 | } 174 | } 175 | p.SetSearchWithParts(pps) 176 | return nil 177 | } 178 | 179 | func (p *Paginate) SetSearchWithParts(parts []string) { 180 | doubleSep := func(s string) string { 181 | return strings.ReplaceAll(s, ";", ";;") 182 | } 183 | pps := []string{} 184 | for _, part := range parts { 185 | pps = append(pps, doubleSep(part)) 186 | } 187 | p.Search = strings.Join(pps, ";") 188 | } 189 | 190 | func (p *Paginate) Namespaces(nss []string) error { 191 | s, err := p.SearchSelector() 192 | if err != nil { 193 | return err 194 | } 195 | nsKey := "namespace" 196 | mes := []v1.LabelSelectorRequirement{ 197 | { 198 | Key: nsKey, 199 | Operator: v1.LabelSelectorOpIn, 200 | Values: nss, 201 | }, 202 | } 203 | for _, r := range s.MatchExpressions { 204 | if r.Key != nsKey { 205 | mes = append(mes, r) 206 | } 207 | } 208 | s.MatchExpressions = mes 209 | return p.SetSearchSelector(s) 210 | } 211 | 212 | func (p *Paginate) Clusters(css []string) error { 213 | s, err := p.SearchSelector() 214 | if err != nil { 215 | return err 216 | } 217 | for _, c := range css { 218 | if c == "" { 219 | return fmt.Errorf("unexpected cluster %q", c) 220 | } 221 | } 222 | nsKey := "cluster" 223 | mes := []v1.LabelSelectorRequirement{ 224 | { 225 | Key: nsKey, 226 | Operator: v1.LabelSelectorOpIn, 227 | Values: css, 228 | }, 229 | } 230 | for _, r := range s.MatchExpressions { 231 | if r.Key != nsKey { 232 | mes = append(mes, r) 233 | } 234 | } 235 | s.MatchExpressions = mes 236 | return p.SetSearchSelector(s) 237 | } 238 | 239 | func (p *Paginate) GetClusters() []string { 240 | if p == nil { 241 | return nil 242 | } 243 | s, err := p.SearchSelector() 244 | if err != nil { 245 | return nil 246 | } 247 | nsKey := "cluster" 248 | for _, r := range s.MatchExpressions { 249 | if r.Key == nsKey { 250 | return r.Values 251 | } 252 | } 253 | return nil 254 | } 255 | 256 | func QueryGetOptions(options v1.GetOptions, cluster string) (v1.GetOptions, error) { 257 | if options.ResourceVersion != "" { 258 | return options, fmt.Errorf("can not set ResourceVersion if wrap cluster for GetOptions") 259 | } 260 | options.ResourceVersion = "dsm-cluster-" + cluster 261 | return options, nil 262 | } 263 | 264 | func QueryCreateOptions(options v1.CreateOptions, cluster string) (v1.CreateOptions, error) { 265 | if options.FieldManager != "" { 266 | return options, fmt.Errorf("can not set ResourceVersion if wrap cluster for CreateOptions") 267 | } 268 | options.FieldManager = "dsm-cluster-" + cluster 269 | return options, nil 270 | } 271 | 272 | func QueryUpdateOptions(options v1.UpdateOptions, cluster string) (v1.UpdateOptions, error) { 273 | if options.FieldManager != "" { 274 | return options, fmt.Errorf("can not set ResourceVersion if wrap cluster for UpdateOptions") 275 | } 276 | options.FieldManager = "dsm-cluster-" + cluster 277 | return options, nil 278 | } 279 | 280 | func QueryPatchOptions(options v1.PatchOptions, cluster string) (v1.PatchOptions, error) { 281 | if len(options.FieldManager) != 0 { 282 | return options, fmt.Errorf("can not set ResourceVersion if wrap cluster for PatchOptions") 283 | } 284 | options.FieldManager = "dsm-cluster-" + cluster 285 | return options, nil 286 | } 287 | 288 | func QueryDeleteOptions(options v1.DeleteOptions, cluster string) (v1.DeleteOptions, error) { 289 | if len(options.DryRun) != 0 { 290 | return options, fmt.Errorf("can not set ResourceVersion if wrap cluster for DeleteOptions") 291 | } 292 | options.DryRun = []string{"dsm-cluster-" + cluster} 293 | return options, nil 294 | } 295 | 296 | func QueryListOptions(options v1.ListOptions, page Paginate) (v1.ListOptions, error) { 297 | bs, _ := json.Marshal(page) 298 | if string(bs) == "{}" { 299 | return options, nil 300 | } 301 | s := base64.StdEncoding.WithPadding(base64.NoPadding).EncodeToString(bs) 302 | //if options.LabelSelector == "" { 303 | // options.LabelSelector = fmt.Sprintf("%s notin (%s)", common.PaginateKey, s) 304 | // return options 305 | //} 306 | ls, err := kube.ParseToLabelSelector(options.LabelSelector) 307 | if err != nil { 308 | return options, err 309 | } 310 | mes := []v1.LabelSelectorRequirement{{ 311 | Key: constants.PaginateKey, 312 | Operator: v1.LabelSelectorOpNotIn, 313 | Values: kube.SplittingValue(s), 314 | }} 315 | for _, m := range ls.MatchExpressions { 316 | if m.Key != constants.PaginateKey { 317 | mes = append(mes, m) 318 | } 319 | } 320 | ls.MatchExpressions = mes 321 | sstr := v1.FormatLabelSelector(ls) 322 | if sstr == "" { 323 | return options, fmt.Errorf("parse selector %v error", ls) 324 | } 325 | options.LabelSelector = sstr 326 | return options, nil 327 | } 328 | 329 | func MakeupResPaginate(l v1.ListInterface, page Paginate) Paginate { 330 | remain := l.GetRemainingItemCount() 331 | val := reflect.ValueOf(l).Elem() 332 | items := 0 333 | for i := 0; i < val.NumField(); i++ { 334 | valueField := val.Field(i) 335 | typeField := val.Type().Field(i) 336 | 337 | f := valueField.Interface() 338 | val := reflect.ValueOf(f) 339 | if typeField.Name == "Items" { 340 | items = val.Len() 341 | } 342 | } 343 | if remain == nil { 344 | var i int64 = 0 345 | remain = &i 346 | } 347 | page.Total = *remain + (page.Page-1)*page.PageSize + int64(items) 348 | return page 349 | } 350 | 351 | func GetObjectCluster(o v1.Object) string { 352 | return o.GetAnnotations()[constants.DSMClusterAnno] 353 | } 354 | -------------------------------------------------------------------------------- /page/page_test.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "fmt" 5 | "github.com/DaoCloud/ckube/common/constants" 6 | "github.com/stretchr/testify/assert" 7 | v12 "k8s.io/api/core/v1" 8 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "testing" 10 | ) 11 | 12 | func TestQueryListOptions(t *testing.T) { 13 | cases := []struct { 14 | name string 15 | inOption v1.ListOptions 16 | inPage Paginate 17 | out v1.ListOptions 18 | wannaError bool 19 | }{ 20 | { 21 | name: "empty Paginate", 22 | }, 23 | { 24 | name: "page", 25 | inPage: Paginate{ 26 | Page: 1, 27 | }, 28 | out: v1.ListOptions{ 29 | LabelSelector: "ckube.daocloud.io/query notin (eyJwYWdlIjoxfQ)", 30 | }, 31 | }, 32 | { 33 | name: "page & page_size", 34 | inPage: Paginate{ 35 | Page: 1, 36 | PageSize: 1, 37 | }, 38 | out: v1.ListOptions{ 39 | LabelSelector: "ckube.daocloud.io/query notin (eyJwYWdlIjoxLCJwYWdlX3NpemUiOjF9)", 40 | }, 41 | }, 42 | { 43 | name: "page & search", 44 | inPage: Paginate{ 45 | Page: 1, 46 | Search: "name=ok", 47 | }, 48 | out: v1.ListOptions{ 49 | LabelSelector: "ckube.daocloud.io/query notin (eyJwYWdlIjoxLCJzZWFyY2giOiJuYW1lPW9rIn0)", 50 | }, 51 | }, 52 | { 53 | name: "label", 54 | inOption: v1.ListOptions{ 55 | LabelSelector: "test=1", 56 | }, 57 | inPage: Paginate{ 58 | Page: 1, 59 | Search: "name=ok", 60 | }, 61 | out: v1.ListOptions{ 62 | LabelSelector: "ckube.daocloud.io/query notin (eyJwYWdlIjoxLCJzZWFyY2giOiJuYW1lPW9rIn0),test=1", 63 | }, 64 | }, 65 | { 66 | name: "multi label", 67 | inOption: v1.ListOptions{ 68 | LabelSelector: "test=1,ckube.daocloud.io/query!=eyJwYWdlIjoxfQ", 69 | }, 70 | inPage: Paginate{ 71 | Page: 1, 72 | Search: "name=ok", 73 | }, 74 | out: v1.ListOptions{ 75 | LabelSelector: "ckube.daocloud.io/query notin (eyJwYWdlIjoxLCJzZWFyY2giOiJuYW1lPW9rIn0),test=1", 76 | }, 77 | }, 78 | } 79 | for i, c := range cases { 80 | t.Run(fmt.Sprintf("%d---%s", i, c.name), func(t *testing.T) { 81 | out, err := QueryListOptions(c.inOption, c.inPage) 82 | assert.Equal(t, c.out, out) 83 | if c.wannaError { 84 | assert.Error(t, err) 85 | } else { 86 | assert.NoError(t, err) 87 | } 88 | }) 89 | } 90 | } 91 | 92 | func TestPaginate_Match(t *testing.T) { 93 | cases := []struct { 94 | name string 95 | index map[string]string 96 | p Paginate 97 | match bool 98 | err error 99 | }{ 100 | { 101 | name: "match fuzzy", 102 | index: map[string]string{ 103 | "name": "test", 104 | "ok": "qq", 105 | }, 106 | p: Paginate{ 107 | Search: "qq", 108 | }, 109 | match: true, 110 | err: nil, 111 | }, 112 | { 113 | name: "match fuzzy 2", 114 | index: map[string]string{ 115 | "name": "test", 116 | "ok": "qq", 117 | }, 118 | p: Paginate{ 119 | Search: "test", 120 | }, 121 | match: true, 122 | err: nil, 123 | }, 124 | { 125 | name: "contains", 126 | index: map[string]string{ 127 | "name": "test", 128 | "ok": "qq", 129 | }, 130 | p: Paginate{ 131 | Search: "name=est", 132 | }, 133 | match: true, 134 | err: nil, 135 | }, 136 | { 137 | name: "full match", 138 | index: map[string]string{ 139 | "name": "test", 140 | "ok": "qq", 141 | }, 142 | p: Paginate{ 143 | Search: "name=\"test\"", 144 | }, 145 | match: true, 146 | err: nil, 147 | }, 148 | { 149 | name: "full match 2", 150 | index: map[string]string{ 151 | "name": "test", 152 | "ok": "qq", 153 | }, 154 | p: Paginate{ 155 | Search: "name=\"est\"", 156 | }, 157 | match: false, 158 | err: nil, 159 | }, 160 | { 161 | name: "advance match", 162 | index: map[string]string{ 163 | "name": "test", 164 | "ok": "qq", 165 | }, 166 | p: Paginate{ 167 | Search: "__ckube_as__: name in (test)", 168 | }, 169 | match: true, 170 | err: nil, 171 | }, 172 | { 173 | name: "advance match not equal", 174 | index: map[string]string{ 175 | "name": "test", 176 | "ok": "qq", 177 | }, 178 | p: Paginate{ 179 | Search: "__ckube_as__: name != test", 180 | }, 181 | match: false, 182 | err: nil, 183 | }, 184 | { 185 | name: "multiple", 186 | index: map[string]string{ 187 | "name": "test", 188 | "ok": "qq", 189 | }, 190 | p: Paginate{ 191 | Search: "ok=qq; __ckube_as__: name=test", 192 | }, 193 | match: true, 194 | err: nil, 195 | }, 196 | { 197 | name: "key error", 198 | index: map[string]string{ 199 | "name": "test", 200 | "ok": "qq", 201 | }, 202 | p: Paginate{ 203 | Search: "xx=test", 204 | }, 205 | match: false, 206 | err: fmt.Errorf("unexpected search key: xx"), 207 | }, 208 | { 209 | name: "not key contains", 210 | index: map[string]string{ 211 | "name": "test", 212 | "ok": "qq", 213 | }, 214 | p: Paginate{ 215 | Search: "name=!test", 216 | }, 217 | match: false, 218 | }, 219 | { 220 | name: "not contains", 221 | index: map[string]string{ 222 | "name": "test", 223 | "ok": "qq", 224 | }, 225 | p: Paginate{ 226 | Search: "!test", 227 | }, 228 | match: false, 229 | }, 230 | { 231 | name: "not contains 2", 232 | index: map[string]string{ 233 | "name": "test", 234 | "ok": "qq", 235 | }, 236 | p: Paginate{ 237 | Search: "!xxx", 238 | }, 239 | match: true, 240 | }, 241 | { 242 | name: "simbol", 243 | index: map[string]string{ 244 | "name": "a;b", 245 | "ok": "qq", 246 | }, 247 | p: Paginate{ 248 | Search: "a;;", 249 | }, 250 | match: true, 251 | }, 252 | { 253 | name: "simbol and not contains", 254 | index: map[string]string{ 255 | "name": "a;b", 256 | "ok": "qq", 257 | }, 258 | p: Paginate{ 259 | Search: "a;;;!qq", 260 | }, 261 | match: false, 262 | }, 263 | } 264 | for i, c := range cases { 265 | t.Run(fmt.Sprintf("%d-%s", i, c.name), func(t *testing.T) { 266 | match, err := c.p.Match(c.index) 267 | assert.Equal(t, c.match, match) 268 | assert.Equal(t, c.err, err) 269 | }) 270 | } 271 | } 272 | 273 | func TestPaginate_Namespaces(t *testing.T) { 274 | p := Paginate{} 275 | err := p.Namespaces([]string{"test", "test1"}) 276 | if err != nil { 277 | t.Fatal(err) 278 | } 279 | assert.Equal(t, "__ckube_as__:namespace in (test,test1)", p.Search) 280 | p = Paginate{ 281 | Search: "test=ok", 282 | } 283 | err = p.Namespaces([]string{"test", "test1"}) 284 | if err != nil { 285 | t.Fatal(err) 286 | } 287 | assert.Equal(t, "__ckube_as__:namespace in (test,test1);test=ok", p.Search) 288 | err = p.Namespaces([]string{}) 289 | if err == nil { 290 | t.Fatal("must be error") 291 | } 292 | } 293 | 294 | func TestPaginate_Clusters(t *testing.T) { 295 | p := Paginate{} 296 | err := p.Clusters([]string{"test", "test1"}) 297 | if err != nil { 298 | t.Fatal(err) 299 | } 300 | assert.Equal(t, "__ckube_as__:cluster in (test,test1)", p.Search) 301 | p = Paginate{ 302 | Search: "test=ok", 303 | } 304 | err = p.Clusters([]string{"test", "test1"}) 305 | if err != nil { 306 | t.Fatal(err) 307 | } 308 | assert.Equal(t, "__ckube_as__:cluster in (test,test1);test=ok", p.Search) 309 | err = p.Clusters([]string{}) 310 | if err == nil { 311 | t.Fatal("must be error") 312 | } 313 | } 314 | 315 | func TestPaginate_GetClusters(t *testing.T) { 316 | p := Paginate{} 317 | err := p.Clusters([]string{"test", "test1"}) 318 | if err != nil { 319 | t.Fatal(err) 320 | } 321 | assert.Equal(t, "__ckube_as__:cluster in (test,test1)", p.Search) 322 | assert.Equal(t, []string{"test", "test1"}, p.GetClusters()) 323 | } 324 | 325 | func TestGetObjectCluster(t *testing.T) { 326 | cases := []struct { 327 | name string 328 | obj v1.Object 329 | cluster string 330 | }{ 331 | { 332 | name: "no annotations", 333 | obj: &v12.Pod{ 334 | ObjectMeta: v1.ObjectMeta{ 335 | Annotations: nil, 336 | }, 337 | }, 338 | cluster: "", 339 | }, 340 | { 341 | name: "default", 342 | obj: &v12.Pod{ 343 | ObjectMeta: v1.ObjectMeta{ 344 | Annotations: map[string]string{ 345 | constants.DSMClusterAnno: "default", 346 | }, 347 | }, 348 | }, 349 | cluster: "default", 350 | }, 351 | { 352 | name: "no cluster", 353 | obj: &v12.Pod{ 354 | ObjectMeta: v1.ObjectMeta{ 355 | Annotations: map[string]string{ 356 | "ok": "default", 357 | }, 358 | }, 359 | }, 360 | cluster: "", 361 | }, 362 | } 363 | for _, c := range cases { 364 | t.Run(c.name, func(t *testing.T) { 365 | assert.Equal(t, c.cluster, GetObjectCluster(c.obj)) 366 | }) 367 | } 368 | } 369 | -------------------------------------------------------------------------------- /pkg/client/fake/fake_ckube.go: -------------------------------------------------------------------------------- 1 | package fake 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | "strings" 12 | "sync" 13 | "time" 14 | 15 | "github.com/gorilla/mux" 16 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 17 | "k8s.io/client-go/rest" 18 | 19 | "github.com/DaoCloud/ckube/common" 20 | "github.com/DaoCloud/ckube/common/constants" 21 | "github.com/DaoCloud/ckube/kube" 22 | "github.com/DaoCloud/ckube/log" 23 | "github.com/DaoCloud/ckube/page" 24 | "github.com/DaoCloud/ckube/server" 25 | "github.com/DaoCloud/ckube/store" 26 | "github.com/DaoCloud/ckube/store/memory" 27 | "github.com/DaoCloud/ckube/watcher" 28 | ) 29 | 30 | type fakeCkubeServer struct { 31 | store store.Store 32 | ser server.Server 33 | kubeConfig *rest.Config 34 | eventChan chan Event 35 | watchChanMap map[string]chan Event 36 | watchChanLock sync.RWMutex 37 | } 38 | 39 | func NewFakeCKubeServer(listenAddr string, config string) (CkubeServer, error) { 40 | cfg := common.Config{} 41 | err := json.Unmarshal([]byte(config), &cfg) 42 | if err != nil { 43 | return nil, err 44 | } 45 | cfg.Token = "" 46 | common.InitConfig(&cfg) 47 | indexConf := map[store.GroupVersionResource]map[string]string{} 48 | for _, proxy := range cfg.Proxies { 49 | indexConf[store.GroupVersionResource{ 50 | Group: proxy.Group, 51 | Version: proxy.Version, 52 | Resource: proxy.Resource, 53 | }] = proxy.Index 54 | } 55 | m := memory.NewMemoryStore(indexConf) 56 | addr := "http://" + func() string { 57 | parts := strings.Split(listenAddr, ":") 58 | if parts[0] == "" { 59 | return listenAddr 60 | } 61 | return "127.0.0.1:" + parts[1] 62 | }() 63 | s := fakeCkubeServer{ 64 | store: m, 65 | eventChan: make(chan Event), 66 | kubeConfig: &rest.Config{ 67 | Host: addr, 68 | }, 69 | watchChanMap: make(map[string]chan Event), 70 | } 71 | ser := server.NewMuxServer(listenAddr, nil, m, s.registerFakeRoute) 72 | s.ser = ser 73 | go ser.Run() // nolint: errcheck 74 | for i := 0; i < 5; i++ { 75 | time.Sleep(time.Millisecond * 100 * time.Duration(1< 0 { 289 | paginateStr, err = kube.MergeValues(m.Values) 290 | if err != nil { 291 | errorProxy(writer, metav1.Status{ 292 | Message: err.Error(), 293 | Code: 400, 294 | }) 295 | return 296 | } 297 | } 298 | } else { 299 | mes = append(mes, m) 300 | } 301 | } 302 | labels.MatchExpressions = mes 303 | } 304 | if paginateStr != "" { 305 | rr, err := base64.StdEncoding.WithPadding(base64.NoPadding).DecodeString(paginateStr) 306 | if err != nil { 307 | errorProxy(writer, metav1.Status{ 308 | Message: err.Error(), 309 | Code: 400, 310 | }) 311 | return 312 | } 313 | _ = json.Unmarshal(rr, &paginate) 314 | delete(labels.MatchLabels, constants.PaginateKey) 315 | } 316 | } 317 | 318 | s.watchChanLock.Lock() 319 | s.watchChanMap[r.RemoteAddr] = make(chan Event) 320 | s.watchChanLock.Unlock() 321 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) 322 | defer cancel() 323 | writer.Header().Set("Content-Type", "application/json") 324 | writer.Header().Set("Transfer-Encoding", "chunked") 325 | writer.Header().Set("Connection", "keep-alive") 326 | writer.(http.Flusher).Flush() 327 | for { 328 | s.watchChanLock.RLock() 329 | wc := s.watchChanMap[r.RemoteAddr] 330 | s.watchChanLock.RUnlock() 331 | select { 332 | case <-ctx.Done(): 333 | s.watchChanLock.Lock() 334 | close(s.watchChanMap[r.RemoteAddr]) 335 | delete(s.watchChanMap, r.RemoteAddr) 336 | s.watchChanLock.Unlock() 337 | return 338 | case e := <-wc: 339 | if e.Group == group && 340 | e.Version == version && 341 | e.Resource == resourceType && 342 | (e.Namespace == namespace || namespace == "") { 343 | 344 | if len(paginate.GetClusters()) > 0 && !func() bool { 345 | for _, c := range paginate.GetClusters() { 346 | if e.Cluster == c { 347 | return true 348 | } 349 | } 350 | return false 351 | }() { 352 | continue 353 | } 354 | typ := "ERROR" 355 | switch e.EventAction { 356 | case EventActionAdd: 357 | typ = "ADDED" 358 | case EventActionDelete: 359 | typ = "DELETED" 360 | case EventActionUpdate: 361 | typ = "MODIFIED" 362 | } 363 | res := fmt.Sprintf(`{"type": %q, "object": %s}`, typ, e.Raw) 364 | _, _ = writer.Write([]byte(res + "\n")) 365 | writer.(http.Flusher).Flush() 366 | } 367 | } 368 | } 369 | } 370 | 371 | func (s *fakeCkubeServer) proxy(writer http.ResponseWriter, r *http.Request) { 372 | group := mux.Vars(r)["group"] 373 | version := mux.Vars(r)["version"] 374 | resourceType := mux.Vars(r)["resourceType"] 375 | namespace := mux.Vars(r)["namespace"] 376 | resourceName := mux.Vars(r)["resourceName"] 377 | gvr := store.GroupVersionResource{ 378 | Group: group, 379 | Version: version, 380 | Resource: resourceType, 381 | } 382 | 383 | query := r.URL.Query() 384 | cluster := common.GetConfig().DefaultCluster 385 | for k, v := range query { 386 | switch k { 387 | case "fieldManager", "resourceVersion": // For Get Create Patch Update actions. 388 | if strings.HasPrefix(v[0], constants.ClusterPrefix) { 389 | cluster = v[0][len(constants.ClusterPrefix):] 390 | } 391 | case "watch": 392 | if strings.ToLower(v[0]) == "true" || strings.ToLower(v[0]) == "1" { 393 | s.watch(writer, r) 394 | return 395 | } 396 | } 397 | } 398 | obj := watcher.ObjType{} 399 | bs, err := io.ReadAll(r.Body) 400 | if err != nil { 401 | errorProxy(writer, metav1.Status{ 402 | Message: err.Error(), 403 | Code: 500, 404 | }) 405 | return 406 | } 407 | _ = json.Unmarshal(bs, &obj) 408 | obj.Namespace = namespace 409 | if resourceName == "" { 410 | resourceName = obj.Name 411 | } 412 | action := EventActionError 413 | switch r.Method { 414 | case "POST": 415 | action = EventActionAdd 416 | if o := s.store.Get(gvr, cluster, namespace, resourceName); o != nil { 417 | errorProxy(writer, metav1.Status{ 418 | Message: fmt.Sprintf("resource %v %s %s/%s already exists", gvr, cluster, namespace, resourceName), 419 | Code: 400, 420 | }) 421 | return 422 | } 423 | _ = s.store.OnResourceAdded(gvr, cluster, &obj) 424 | case "PUT": 425 | action = EventActionUpdate 426 | if o := s.store.Get(gvr, cluster, namespace, resourceName); o == nil { 427 | errorProxy(writer, metav1.Status{ 428 | Message: fmt.Sprintf("resource %v %s %s/%s not found", gvr, cluster, namespace, resourceName), 429 | Code: 404, 430 | }) 431 | return 432 | } 433 | _ = s.store.OnResourceModified(gvr, cluster, &obj) 434 | case "DELETE": 435 | action = EventActionDelete 436 | del := metav1.DeleteOptions{} 437 | _ = json.Unmarshal(bs, &del) 438 | if len(del.DryRun) == 1 && strings.HasPrefix(del.DryRun[0], constants.ClusterPrefix) { 439 | cluster = del.DryRun[0][len(constants.ClusterPrefix):] 440 | } 441 | if o := s.store.Get(gvr, cluster, namespace, resourceName); o == nil { 442 | errorProxy(writer, metav1.Status{ 443 | Message: fmt.Sprintf("resource %v %s %s/%s not found", gvr, cluster, namespace, resourceName), 444 | Code: 404, 445 | }) 446 | return 447 | } else { 448 | bs, _ = json.Marshal(o) 449 | } 450 | obj.Name = resourceName 451 | obj.Namespace = namespace 452 | _ = s.store.OnResourceDeleted(gvr, cluster, &obj) 453 | } 454 | e := Event{ 455 | EventAction: action, 456 | Group: group, 457 | Version: version, 458 | Resource: resourceType, 459 | Cluster: cluster, 460 | Namespace: namespace, 461 | Name: resourceName, 462 | Raw: string(bs), 463 | } 464 | select { 465 | case s.eventChan <- e: 466 | default: 467 | } 468 | s.watchChanLock.RLock() 469 | for remote, c := range s.watchChanMap { 470 | select { 471 | case c <- e: 472 | log.Debugf("succeed send stream to %s", remote) 473 | default: 474 | log.Infof("remote watcher %s no active stream", remote) 475 | } 476 | } 477 | s.watchChanLock.RUnlock() 478 | jsonResp(writer, 200, obj) 479 | } 480 | -------------------------------------------------------------------------------- /pkg/client/fake/fake_client_test.go: -------------------------------------------------------------------------------- 1 | package fake 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "sync" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | v1 "k8s.io/api/core/v1" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/client-go/kubernetes" 14 | 15 | "github.com/DaoCloud/ckube/page" 16 | ) 17 | 18 | func TestNewFakeCKubeServer(t *testing.T) { 19 | s, err := NewFakeCKubeServer(":65521", ` 20 | { 21 | "proxies": [ 22 | { 23 | "group": "", 24 | "version": "v1", 25 | "resource": "pods", 26 | "list_kind": "PodList", 27 | "index": { 28 | "namespace": "{.metadata.namespace}", 29 | "name": "{.metadata.name}", 30 | "labels": "{.metadata.labels}", 31 | "created_at": "{.metadata.creationTimestamp}" 32 | } 33 | }, 34 | { 35 | "group": "", 36 | "version": "v1", 37 | "resource": "services", 38 | "list_kind": "ServiceList", 39 | "index": { 40 | "namespace": "{.metadata.namespace}", 41 | "name": "{.metadata.name}", 42 | "labels": "{.metadata.labels}", 43 | "created_at": "{.metadata.creationTimestamp}" 44 | } 45 | } 46 | ] 47 | } 48 | `) 49 | assert.NoError(t, err) 50 | defer s.Stop() 51 | cfb := s.GetKubeConfig() 52 | cli, err := kubernetes.NewForConfig(cfb) 53 | assert.NoError(t, err) 54 | t.Run("create pods", func(t *testing.T) { 55 | coptc1, _ := page.QueryCreateOptions(metav1.CreateOptions{}, "c1") 56 | coptc2, _ := page.QueryCreateOptions(metav1.CreateOptions{}, "c2") 57 | _, err = cli.CoreV1().Pods("test").Create(context.Background(), &v1.Pod{ 58 | TypeMeta: metav1.TypeMeta{}, 59 | ObjectMeta: metav1.ObjectMeta{ 60 | Name: "pod1", 61 | Namespace: "test", 62 | }, 63 | Spec: v1.PodSpec{ 64 | DNSPolicy: "ClusterFirst", 65 | }, 66 | }, coptc1) 67 | assert.NoError(t, err) 68 | _, err = cli.CoreV1().Pods("test").Create(context.Background(), &v1.Pod{ 69 | TypeMeta: metav1.TypeMeta{}, 70 | ObjectMeta: metav1.ObjectMeta{ 71 | Name: "pod2", 72 | Namespace: "test", 73 | }, 74 | Spec: v1.PodSpec{ 75 | DNSPolicy: "ClusterFirst", 76 | }, 77 | }, coptc2) 78 | assert.NoError(t, err) 79 | }) 80 | t.Run("get pod each cluster", func(t *testing.T) { 81 | goptc1, _ := page.QueryGetOptions(metav1.GetOptions{}, "c1") 82 | goptc2, _ := page.QueryGetOptions(metav1.GetOptions{}, "c2") 83 | p1, err := cli.CoreV1().Pods("test").Get(context.Background(), "pod1", goptc1) 84 | assert.NoError(t, err) 85 | assert.Equal(t, v1.DNSPolicy("ClusterFirst"), p1.Spec.DNSPolicy) 86 | _, err = cli.CoreV1().Pods("test").Get(context.Background(), "pod1", goptc2) 87 | assert.Error(t, err) 88 | }) 89 | t.Run("list pods", func(t *testing.T) { 90 | p := page.Paginate{} 91 | _ = p.Clusters([]string{"c1", "c2"}) 92 | lopts, _ := page.QueryListOptions(metav1.ListOptions{}, p) 93 | pods, err := cli.CoreV1().Pods("test").List(context.Background(), lopts) 94 | assert.NoError(t, err) 95 | assert.Len(t, pods.Items, 2) 96 | }) 97 | t.Run("update pods", func(t *testing.T) { 98 | uoptc1, _ := page.QueryUpdateOptions(metav1.UpdateOptions{}, "c1") 99 | _, err := cli.CoreV1().Pods("test").Update(context.Background(), &v1.Pod{ 100 | TypeMeta: metav1.TypeMeta{}, 101 | ObjectMeta: metav1.ObjectMeta{ 102 | Name: "pod1", 103 | Namespace: "test", 104 | }, 105 | Spec: v1.PodSpec{ 106 | DNSPolicy: "Default", 107 | }, 108 | }, uoptc1) 109 | assert.NoError(t, err) 110 | goptc1, _ := page.QueryGetOptions(metav1.GetOptions{}, "c1") 111 | p1, err := cli.CoreV1().Pods("test").Get(context.Background(), "pod1", goptc1) 112 | assert.NoError(t, err) 113 | assert.Equal(t, v1.DNSPolicy("Default"), p1.Spec.DNSPolicy) 114 | }) 115 | t.Run("delete pods", func(t *testing.T) { 116 | doptc1, _ := page.QueryDeleteOptions(metav1.DeleteOptions{}, "c1") 117 | err := cli.CoreV1().Pods("test").Delete(context.Background(), "pod1", doptc1) 118 | assert.NoError(t, err) 119 | goptc1, _ := page.QueryGetOptions(metav1.GetOptions{}, "c1") 120 | _, err = cli.CoreV1().Pods("test").Get(context.Background(), "pod1", goptc1) 121 | assert.Error(t, err) 122 | }) 123 | 124 | t.Run("events", func(t *testing.T) { 125 | events := []Event{} 126 | lock := sync.Mutex{} 127 | go func() { 128 | for e := range s.Events() { 129 | lock.Lock() 130 | events = append(events, e) 131 | lock.Unlock() 132 | } 133 | }() 134 | coptc1, _ := page.QueryCreateOptions(metav1.CreateOptions{}, "c1") 135 | _, err = cli.CoreV1().Pods("test").Create(context.Background(), &v1.Pod{ 136 | TypeMeta: metav1.TypeMeta{}, 137 | ObjectMeta: metav1.ObjectMeta{ 138 | Name: "pod1", 139 | Namespace: "test", 140 | }, 141 | Spec: v1.PodSpec{ 142 | DNSPolicy: "ClusterFirst", 143 | }, 144 | }, coptc1) 145 | assert.NoError(t, err) 146 | lock.Lock() 147 | defer lock.Unlock() 148 | assert.Equal(t, []Event{ 149 | { 150 | EventAction: EventActionAdd, 151 | Group: "", 152 | Version: "v1", 153 | Resource: "pods", 154 | Cluster: "c1", 155 | Namespace: "test", 156 | Name: "pod1", 157 | Raw: "{\"kind\":\"Pod\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"pod1\",\"namespace\":\"test\",\"creationTimestamp\":null},\"spec\":{\"containers\":null,\"dnsPolicy\":\"ClusterFirst\"},\"status\":{}}\n", 158 | }, 159 | }, events) 160 | }) 161 | t.Run("watch", func(t *testing.T) { 162 | events := []Event{} 163 | lock := sync.Mutex{} 164 | wg := sync.WaitGroup{} 165 | wg.Add(1) 166 | go func() { 167 | p := page.Paginate{} 168 | _ = p.Clusters([]string{"c1"}) 169 | lopts, _ := page.QueryListOptions(metav1.ListOptions{}, p) 170 | w, err := cli.CoreV1().Pods("test").Watch(context.Background(), lopts) 171 | assert.NoError(t, err) 172 | wg.Done() 173 | for e := range w.ResultChan() { 174 | pod := e.Object.(*v1.Pod) 175 | lock.Lock() 176 | events = append(events, Event{ 177 | Cluster: page.GetObjectCluster(pod), 178 | Namespace: pod.Namespace, 179 | Name: pod.Name, 180 | }) 181 | lock.Unlock() 182 | } 183 | }() 184 | wg.Wait() 185 | coptc1, _ := page.QueryCreateOptions(metav1.CreateOptions{}, "c1") 186 | _, err = cli.CoreV1().Pods("test").Create(context.Background(), &v1.Pod{ 187 | TypeMeta: metav1.TypeMeta{}, 188 | ObjectMeta: metav1.ObjectMeta{ 189 | Name: "pod3", 190 | Namespace: "test", 191 | }, 192 | Spec: v1.PodSpec{ 193 | DNSPolicy: "ClusterFirst", 194 | }, 195 | }, coptc1) 196 | coptc2, _ := page.QueryCreateOptions(metav1.CreateOptions{}, "c2") 197 | _, err = cli.CoreV1().Pods("test").Create(context.Background(), &v1.Pod{ 198 | TypeMeta: metav1.TypeMeta{}, 199 | ObjectMeta: metav1.ObjectMeta{ 200 | Name: "pod4", 201 | Namespace: "test", 202 | }, 203 | Spec: v1.PodSpec{ 204 | DNSPolicy: "ClusterFirst", 205 | }, 206 | }, coptc2) 207 | assert.NoError(t, err) 208 | lock.Lock() 209 | defer lock.Unlock() 210 | assert.Equal(t, []Event{ 211 | { 212 | Namespace: "test", 213 | Name: "pod3", 214 | }, 215 | }, events) 216 | }) 217 | t.Run("custom api", func(t *testing.T) { 218 | _, _ = cli.CoreV1().Pods("test").Create(context.Background(), &v1.Pod{ 219 | ObjectMeta: metav1.ObjectMeta{ 220 | Name: "test-xxxx-asd", 221 | Namespace: "test", 222 | Labels: map[string]string{ 223 | "app": "test", 224 | }, 225 | OwnerReferences: []metav1.OwnerReference{ 226 | { 227 | Kind: "ReplicaSet", 228 | Name: "test-xxxx", 229 | }, 230 | }, 231 | }, 232 | }, metav1.CreateOptions{}) 233 | _, _ = cli.CoreV1().Services("test").Create(context.Background(), &v1.Service{ 234 | ObjectMeta: metav1.ObjectMeta{ 235 | Name: "test-svc", 236 | Namespace: "test", 237 | Labels: map[string]string{ 238 | "app": "test", 239 | }, 240 | }, 241 | Spec: v1.ServiceSpec{ 242 | Ports: []v1.ServicePort{ 243 | { 244 | Port: 20880, 245 | }, 246 | }, 247 | Selector: map[string]string{ 248 | "app": "test", 249 | }, 250 | ClusterIP: "1.1.1.1", 251 | }, 252 | }, metav1.CreateOptions{}) 253 | bs, err := cli.Discovery().RESTClient(). 254 | Get(). 255 | RequestURI(fmt.Sprintf("/custom/v1/namespaces/%s/deployments/%s/services", 256 | // m.Cluster, 257 | "test", 258 | "test")). 259 | DoRaw(context.Background()) 260 | assert.NoError(t, err) 261 | svcs := make([]v1.Service, 0) 262 | err = json.Unmarshal(bs, &svcs) 263 | assert.NoError(t, err) 264 | assert.Len(t, svcs, 1) 265 | }) 266 | } 267 | -------------------------------------------------------------------------------- /pkg/client/fake/interface.go: -------------------------------------------------------------------------------- 1 | package fake 2 | 3 | import "k8s.io/client-go/rest" 4 | 5 | type EventAction int 6 | 7 | const ( 8 | EventActionError EventAction = 0 9 | EventActionAdd EventAction = 1 10 | EventActionUpdate EventAction = 2 11 | EventActionDelete EventAction = 3 12 | ) 13 | 14 | type Event struct { 15 | EventAction 16 | Group string 17 | Version string 18 | Resource string 19 | Cluster string 20 | Namespace string 21 | Name string 22 | Raw string 23 | } 24 | 25 | type CkubeServer interface { 26 | Events() <-chan Event 27 | GetKubeConfig() *rest.Config 28 | Stop() 29 | Clean() 30 | } 31 | -------------------------------------------------------------------------------- /server/route.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/DaoCloud/ckube/api" 5 | "github.com/DaoCloud/ckube/api/extend" 6 | "github.com/DaoCloud/ckube/utils/prommonitor" 7 | ) 8 | 9 | type HandleFunc func(r *api.ReqContext) interface{} 10 | 11 | type route struct { 12 | path string 13 | method string 14 | handler HandleFunc 15 | authRequired bool 16 | successStatus int 17 | prefix bool 18 | } 19 | 20 | var ( 21 | routeHandles = []route{ 22 | // healthy 23 | { 24 | path: "/healthy", 25 | method: "GET", 26 | handler: func(r *api.ReqContext) interface{} { 27 | _, _ = r.Writer.Write([]byte("1")) 28 | r.Writer.WriteHeader(200) 29 | return nil 30 | }, 31 | }, 32 | // metrics url 33 | { 34 | path: "/metrics", 35 | method: "GET", 36 | handler: prommonitor.PromHandler, 37 | }, 38 | { 39 | path: "/custom/v1/namespaces/{namespace}/deployments/{deployment}/services", 40 | method: "GET", 41 | handler: extend.Deploy2Service, 42 | authRequired: true, 43 | successStatus: 200, 44 | }, 45 | { 46 | path: "/custom/v1/clusters/{cluster}/namespaces/{namespace}/deployments/{deployment}/services", 47 | method: "GET", 48 | handler: extend.Deploy2Service, 49 | authRequired: true, 50 | successStatus: 200, 51 | }, 52 | { 53 | path: "/apis/{group}/{version}/namespaces/{namespace}/{resourceType}", 54 | method: "GET", 55 | handler: api.Proxy, 56 | authRequired: true, 57 | successStatus: 200, 58 | }, 59 | { 60 | path: "/apis/{group}/{version}/{resourceType}", 61 | handler: api.Proxy, 62 | authRequired: true, 63 | successStatus: 200, 64 | }, 65 | { 66 | path: "/api/{version}/{resourceType}", 67 | handler: api.Proxy, 68 | authRequired: true, 69 | successStatus: 200, 70 | }, 71 | { 72 | path: "/api/{version}/namespaces/{namespace}/{resourceType}", 73 | handler: api.Proxy, 74 | authRequired: true, 75 | successStatus: 200, 76 | }, 77 | 78 | // single resources 79 | { 80 | path: "/apis/{group}/{version}/namespaces/{namespace}/{resourceType}/{resource}", 81 | handler: api.Proxy, 82 | authRequired: true, 83 | successStatus: 200, 84 | }, 85 | { 86 | path: "/apis/{group}/{version}/{resourceType}/{resource}", 87 | handler: api.Proxy, 88 | authRequired: true, 89 | successStatus: 200, 90 | }, 91 | { 92 | path: "/api/{version}/{resourceType}/{resource}", 93 | handler: api.Proxy, 94 | authRequired: true, 95 | successStatus: 200, 96 | }, 97 | { 98 | path: "/api/{version}/namespaces/{namespace}/{resourceType}/{resource}", 99 | handler: api.Proxy, 100 | authRequired: true, 101 | successStatus: 200, 102 | }, 103 | { 104 | path: "/", 105 | prefix: true, 106 | handler: api.Proxy, 107 | authRequired: true, 108 | successStatus: 200, 109 | }, 110 | } 111 | ) 112 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "runtime/debug" 9 | "strings" 10 | "time" 11 | 12 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/client-go/kubernetes" 14 | 15 | "github.com/DaoCloud/ckube/api" 16 | "github.com/DaoCloud/ckube/common" 17 | "github.com/DaoCloud/ckube/log" 18 | "github.com/DaoCloud/ckube/store" 19 | 20 | "github.com/gorilla/mux" 21 | "github.com/prometheus/client_golang/prometheus/promhttp" 22 | "github.com/sirupsen/logrus" 23 | ) 24 | 25 | type Server interface { 26 | Run() error 27 | Stop() error 28 | ResetStore(store store.Store, clis map[string]kubernetes.Interface) 29 | } 30 | 31 | type muxServer struct { 32 | LogLevel string 33 | ListenAddr string 34 | router *mux.Router 35 | server *http.Server 36 | store store.Store 37 | clusterClients map[string]kubernetes.Interface 38 | } 39 | 40 | type statusWriter struct { 41 | http.ResponseWriter 42 | status int 43 | length int 44 | } 45 | 46 | func (w *statusWriter) WriteHeader(status int) { 47 | w.status = status 48 | w.ResponseWriter.WriteHeader(status) 49 | w.ResponseWriter.(http.Flusher).Flush() 50 | } 51 | 52 | func (w *statusWriter) Write(b []byte) (int, error) { 53 | if w.status == 0 { 54 | w.status = 200 55 | } 56 | n, err := w.ResponseWriter.Write(b) 57 | w.ResponseWriter.(http.Flusher).Flush() 58 | w.length += n 59 | return n, err 60 | } 61 | 62 | func loggingMiddleware(next http.Handler) http.Handler { 63 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 64 | st := time.Now() 65 | sw := statusWriter{ 66 | ResponseWriter: w, 67 | status: 0, 68 | length: 0, 69 | } 70 | next.ServeHTTP(&sw, r) 71 | log.AccessLog.WithFields(logrus.Fields{ 72 | "method": r.Method, 73 | "type": "access", 74 | "path": r.RequestURI, 75 | "req_time": time.Since(st), 76 | "status": sw.status, 77 | "content_length": sw.length, 78 | }).Print() 79 | }) 80 | } 81 | 82 | func NewMuxServer(listenAddr string, clusterClients map[string]kubernetes.Interface, s store.Store, externalRouter ...func(*mux.Router)) Server { 83 | ser := muxServer{ 84 | clusterClients: clusterClients, 85 | store: s, 86 | ListenAddr: listenAddr, 87 | router: mux.NewRouter(), 88 | } 89 | for _, h := range externalRouter { 90 | h(ser.router) 91 | } 92 | ser.registerRoutes(ser.router, routeHandles) 93 | ser.router.Use(loggingMiddleware) 94 | ser.router.HandleFunc("/metrics", promhttp.Handler().ServeHTTP).Methods("GET") 95 | return &ser 96 | } 97 | 98 | func (m *muxServer) Run() error { 99 | m.server = &http.Server{ 100 | Addr: m.ListenAddr, 101 | Handler: m.router, 102 | ReadTimeout: 30 * time.Minute, 103 | WriteTimeout: 30 * time.Minute, 104 | } 105 | log.Infof("starting server at %v", m.ListenAddr) 106 | return m.server.ListenAndServe() 107 | } 108 | 109 | func (m *muxServer) Stop() error { 110 | if m.server == nil { 111 | return fmt.Errorf("server not start ever") 112 | } 113 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) 114 | defer cancel() 115 | log.Infof("shutting down the server...") 116 | return m.server.Shutdown(ctx) 117 | } 118 | 119 | func (m *muxServer) ResetStore(s store.Store, clis map[string]kubernetes.Interface) { 120 | m.store = s 121 | m.clusterClients = clis 122 | } 123 | 124 | func jsonResp(writer http.ResponseWriter, status int, v interface{}) { 125 | b, _ := json.Marshal(v) 126 | writer.Header().Set("Content-Type", "application/json") 127 | writer.WriteHeader(status) 128 | _, _ = writer.Write(b) 129 | } 130 | 131 | func (m *muxServer) registerRoutes(router *mux.Router, handleRoutes []route) { 132 | for _, r := range handleRoutes { 133 | func(route route) { 134 | var rt *mux.Route 135 | if route.prefix { 136 | rt = router.PathPrefix(route.path) 137 | } else { 138 | rt = router.Path(route.path) 139 | if r.method != "" { 140 | rt = rt.Methods(route.method) 141 | } else { 142 | rt = rt.Methods("GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD") 143 | } 144 | } 145 | rt.HandlerFunc(func(writer http.ResponseWriter, r *http.Request) { 146 | defer func() { 147 | // deal 500 error 148 | if err := recover(); err != nil { 149 | log.Errorf("%s:%s request error: %v", r.Method, route.path, err) 150 | debug.PrintStack() 151 | jsonResp(writer, http.StatusInternalServerError, err) 152 | } 153 | }() 154 | if route.authRequired && common.GetConfig().Token != "" { 155 | if !strings.Contains(r.Header.Get("Authorization"), common.GetConfig().Token) { 156 | jsonResp(writer, http.StatusUnauthorized, v1.Status{ 157 | Status: string(v1.StatusReasonUnauthorized), 158 | Message: "token missing or error", 159 | Reason: v1.StatusReason("token missing or error"), 160 | Code: 401, 161 | }) 162 | return 163 | } 164 | } 165 | var res = route.handler(&api.ReqContext{ 166 | ClusterClients: m.clusterClients, 167 | Store: m.store, 168 | Request: r, 169 | Writer: writer, 170 | }) 171 | if res == nil { 172 | return 173 | } 174 | var status int 175 | switch res := res.(type) { 176 | case error: 177 | log.Errorf("request return a unexpected error: %v", res) 178 | panic(res) 179 | case v1.Status: 180 | status = int(res.Code) 181 | case *v1.Status: 182 | status = int(res.Code) 183 | case string: 184 | _, _ = writer.Write([]byte(res)) 185 | return 186 | case []byte: 187 | _, _ = writer.Write(res) 188 | return 189 | default: 190 | status = route.successStatus 191 | } 192 | jsonResp(writer, status, res) 193 | }) 194 | }(r) 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /store/interface.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "github.com/DaoCloud/ckube/page" 5 | ) 6 | 7 | type Filter func(obj Object) (bool, error) 8 | type Sort func(i, j int) bool 9 | 10 | type Query struct { 11 | Namespace string 12 | page.Paginate 13 | } 14 | 15 | type Store interface { 16 | IsStoreGVR(gvr GroupVersionResource) bool 17 | Clean(gvr GroupVersionResource, cluster string) error 18 | OnResourceAdded(gvr GroupVersionResource, cluster string, obj interface{}) error 19 | OnResourceModified(gvr GroupVersionResource, cluster string, obj interface{}) error 20 | OnResourceDeleted(gvr GroupVersionResource, cluster string, obj interface{}) error 21 | Query(gvr GroupVersionResource, query Query) QueryResult 22 | Get(gvr GroupVersionResource, cluster string, namespace, name string) interface{} 23 | } 24 | -------------------------------------------------------------------------------- /store/memory/memory.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "sort" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "text/template" 13 | 14 | "github.com/samber/lo" 15 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/client-go/util/jsonpath" 17 | 18 | "github.com/DaoCloud/ckube/common/constants" 19 | "github.com/DaoCloud/ckube/log" 20 | "github.com/DaoCloud/ckube/store" 21 | "github.com/DaoCloud/ckube/utils" 22 | "github.com/DaoCloud/ckube/utils/prommonitor" 23 | ) 24 | 25 | type syncResourceStore[K comparable, V any] struct { 26 | lock sync.RWMutex 27 | resources map[K]*V 28 | } 29 | 30 | func (s *syncResourceStore[K, V]) Set(key K, value V) { 31 | s.lock.Lock() 32 | if s.resources == nil { 33 | s.resources = make(map[K]*V) 34 | } 35 | s.resources[key] = &value 36 | s.lock.Unlock() 37 | } 38 | 39 | func (s *syncResourceStore[K, V]) Init(key K) { 40 | s.lock.Lock() 41 | defer s.lock.Unlock() 42 | if s.resources == nil { 43 | s.resources = make(map[K]*V) 44 | } 45 | if _, ok := s.resources[key]; ok { 46 | return 47 | } 48 | s.resources[key] = new(V) 49 | } 50 | 51 | func (s *syncResourceStore[K, V]) Clean() { 52 | s.lock.Lock() 53 | defer s.lock.Unlock() 54 | s.resources = make(map[K]*V) 55 | } 56 | 57 | func (s *syncResourceStore[K, V]) Get(key K) *V { 58 | s.lock.RLock() 59 | defer s.lock.RUnlock() 60 | return s.resources[key] 61 | } 62 | 63 | func (s *syncResourceStore[K, V]) Exists(key K) bool { 64 | s.lock.RLock() 65 | defer s.lock.RUnlock() 66 | _, ok := s.resources[key] 67 | return ok 68 | } 69 | 70 | func (s *syncResourceStore[K, V]) Delete(key K) { 71 | s.lock.Lock() 72 | defer s.lock.Unlock() 73 | delete(s.resources, key) 74 | } 75 | 76 | func (s *syncResourceStore[K, V]) Values() []*V { 77 | s.lock.RLock() 78 | defer s.lock.RUnlock() 79 | return lo.Values(s.resources) 80 | } 81 | 82 | func (s *syncResourceStore[K, V]) ForEach(iter func(k K, v *V)) { 83 | s.lock.RLock() 84 | defer s.lock.RUnlock() 85 | for k, v := range s.resources { 86 | iter(k, v) 87 | } 88 | } 89 | 90 | func (s *syncResourceStore[K, V]) Len() int { 91 | s.lock.RLock() 92 | defer s.lock.RUnlock() 93 | return len(s.resources) 94 | } 95 | 96 | type clusterName string 97 | 98 | type namespaceName string 99 | 100 | type memoryStore struct { 101 | indexConf map[store.GroupVersionResource]map[string]string 102 | // resourceMap gvr - clusterName - namespaceName - name 103 | resourceMap syncResourceStore[ 104 | store.GroupVersionResource, 105 | syncResourceStore[ 106 | clusterName, 107 | syncResourceStore[ 108 | namespaceName, 109 | syncResourceStore[string, store.Object], 110 | ], 111 | ], 112 | ] 113 | store.Store 114 | } 115 | 116 | func NewMemoryStore(indexConf map[store.GroupVersionResource]map[string]string) store.Store { 117 | s := memoryStore{ 118 | indexConf: indexConf, 119 | } 120 | for k := range indexConf { 121 | s.resourceMap.Init(k) 122 | } 123 | return &s 124 | } 125 | 126 | func (m *memoryStore) initResourceNamespace(gvr store.GroupVersionResource, cluster, namespace string) { 127 | m.resourceMap.Get(gvr).Init(clusterName(cluster)) 128 | m.resourceMap.Get(gvr).Get(clusterName(cluster)).Init(namespaceName(namespace)) 129 | } 130 | 131 | func (m *memoryStore) IsStoreGVR(gvr store.GroupVersionResource) bool { 132 | return m.resourceMap.Exists(gvr) 133 | } 134 | 135 | func (m *memoryStore) Clean(gvr store.GroupVersionResource, cluster string) error { 136 | if !m.resourceMap.Get(gvr).Exists(clusterName(cluster)) { 137 | return fmt.Errorf("cluster %s not exists", cluster) 138 | } 139 | for _, c := range m.resourceMap.Get(gvr).Get(clusterName(cluster)).Values() { 140 | c.Clean() 141 | } 142 | return nil 143 | } 144 | 145 | func (m *memoryStore) OnResourceAdded(gvr store.GroupVersionResource, cluster string, obj interface{}) error { 146 | ns, name, o := m.buildResourceWithIndex(gvr, cluster, obj) 147 | m.initResourceNamespace(gvr, cluster, ns) 148 | m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(ns)).Set(name, o) 149 | prommonitor.Resources.WithLabelValues(cluster, gvr.Group, gvr.Version, gvr.Resource, ns). 150 | Set(float64(m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(ns)).Len())) 151 | return nil 152 | } 153 | 154 | func (m *memoryStore) OnResourceModified(gvr store.GroupVersionResource, cluster string, obj interface{}) error { 155 | ns, name, o := m.buildResourceWithIndex(gvr, cluster, obj) 156 | m.initResourceNamespace(gvr, cluster, ns) 157 | m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(ns)).Set(name, o) 158 | prommonitor.Resources.WithLabelValues(cluster, gvr.Group, gvr.Version, gvr.Resource, ns). 159 | Set(float64(m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(ns)).Len())) 160 | return nil 161 | } 162 | 163 | func (m *memoryStore) OnResourceDeleted(gvr store.GroupVersionResource, cluster string, obj interface{}) error { 164 | ns, name, _ := m.buildResourceWithIndex(gvr, cluster, obj) 165 | m.initResourceNamespace(gvr, cluster, ns) 166 | m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(ns)).Delete(name) 167 | prommonitor.Resources.WithLabelValues(cluster, gvr.Group, gvr.Version, gvr.Resource, ns). 168 | Set(float64(m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(ns)).Len())) 169 | return nil 170 | } 171 | 172 | type innerSort struct { 173 | key string 174 | typ string 175 | reverse bool 176 | } 177 | 178 | func sortObjs(objs []store.Object, s string) ([]store.Object, error) { 179 | if s == "" { 180 | s = "cluster, namespace, name" 181 | } 182 | if len(objs) == 0 { 183 | return objs, nil 184 | } 185 | checkKeyMap := objs[0].Index 186 | ss := strings.Split(s, ",") 187 | sorts := make([]innerSort, 0, len(ss)) 188 | for _, s = range ss { 189 | s = strings.TrimSpace(s) 190 | if s == "" { 191 | continue 192 | } 193 | st := innerSort{ 194 | reverse: false, 195 | typ: constants.KeyTypeStr, 196 | } 197 | if strings.Contains(s, " ") { 198 | parts := strings.Split(s, " ") 199 | if len(parts) > 2 { 200 | return objs, nil 201 | } 202 | if len(parts) == 2 { 203 | switch parts[1] { 204 | case constants.SortDesc: 205 | st.reverse = true 206 | case constants.SortASC: 207 | st.reverse = false 208 | default: 209 | return objs, fmt.Errorf("error sort format `%s`", parts[1]) 210 | } 211 | } 212 | // override s 213 | s = parts[0] 214 | } 215 | if strings.Contains(s, constants.KeyTypeSep) { 216 | parts := strings.Split(s, constants.KeyTypeSep) 217 | if len(parts) != 2 { 218 | return objs, fmt.Errorf("error type format") 219 | } 220 | switch parts[1] { 221 | case constants.KeyTypeInt: 222 | st.typ = constants.KeyTypeInt 223 | case constants.KeyTypeStr: 224 | st.typ = constants.KeyTypeStr 225 | default: 226 | return objs, fmt.Errorf("unsupported typ: %s", parts[1]) 227 | } 228 | s = parts[0] 229 | } 230 | st.key = s 231 | if _, ok := checkKeyMap[s]; !ok { 232 | return objs, fmt.Errorf("unexpected sort key: %s", s) 233 | } 234 | sorts = append(sorts, st) 235 | } 236 | var sortErr error = nil 237 | sort.Slice(objs, func(i, j int) bool { 238 | for _, s := range sorts { 239 | r := false 240 | equals := false 241 | vis := objs[i].Index[s.key] 242 | vjs := objs[j].Index[s.key] 243 | if s.typ == constants.KeyTypeInt { 244 | keyErr := fmt.Errorf("value of `%s` can not convert to number", s.key) 245 | vi, err := strconv.ParseFloat(vis, 64) 246 | if err != nil { 247 | sortErr = keyErr 248 | break 249 | } 250 | vj, err := strconv.ParseFloat(vjs, 64) 251 | if err != nil { 252 | sortErr = keyErr 253 | break 254 | } 255 | r = vi < vj 256 | equals = vi == vj 257 | } else { 258 | r = vis < vjs 259 | equals = vis == vjs 260 | } 261 | if equals { 262 | continue 263 | } 264 | if s.reverse { 265 | r = !r 266 | } 267 | return r 268 | } 269 | return true 270 | }) 271 | return objs, sortErr 272 | } 273 | 274 | func (m *memoryStore) Get(gvr store.GroupVersionResource, cluster string, namespace, name string) interface{} { 275 | if m.resourceMap.Exists(gvr) { 276 | if !m.resourceMap.Get(gvr).Exists(clusterName(cluster)) { 277 | return nil 278 | } 279 | if !m.resourceMap.Get(gvr).Get(clusterName(cluster)).Exists(namespaceName(namespace)) { 280 | return nil 281 | } 282 | if o := m.resourceMap.Get(gvr).Get(clusterName(cluster)).Get(namespaceName(namespace)).Get(name); o != nil { 283 | return o.Obj 284 | } 285 | } 286 | return nil 287 | } 288 | 289 | func (m *memoryStore) Query(gvr store.GroupVersionResource, query store.Query) store.QueryResult { 290 | res := store.QueryResult{} 291 | resources := make([]store.Object, 0) 292 | m.resourceMap.Get(gvr).ForEach(func(cname clusterName, c *syncResourceStore[ 293 | namespaceName, 294 | syncResourceStore[string, store.Object], 295 | ]) { 296 | c.ForEach(func(ns namespaceName, nssObj *syncResourceStore[string, store.Object]) { 297 | if query.Namespace == "" || query.Namespace == string(ns) { 298 | nssObj.ForEach(func(name string, obj *store.Object) { 299 | if ok, err := query.Match(obj.Index); ok { 300 | resources = append(resources, *obj) 301 | } else if err != nil { 302 | res.Error = err 303 | } 304 | }) 305 | } 306 | }) 307 | }) 308 | l := int64(len(resources)) 309 | if l == 0 { 310 | return res 311 | } 312 | resources, err := sortObjs(resources, query.Sort) 313 | if err != nil { 314 | res.Error = err 315 | return res 316 | } 317 | res.Total = l 318 | var start, end int64 319 | if query.PageSize == 0 { 320 | // all resources 321 | start = 0 322 | end = l 323 | } else { 324 | start = (query.Page - 1) * query.PageSize 325 | end = start + query.PageSize 326 | if start >= l { 327 | start = l 328 | } 329 | if end >= l { 330 | end = l 331 | } 332 | } 333 | for _, r := range resources[start:end] { 334 | res.Items = append(res.Items, r.Obj) 335 | } 336 | return res 337 | } 338 | 339 | var funMap = map[string]interface{}{ 340 | "default": func(def string, pre interface{}) string { 341 | if pre == nil { 342 | return def 343 | } 344 | return fmt.Sprintf("%s", pre) 345 | }, 346 | "quote": func(pre interface{}) string { 347 | return fmt.Sprintf("%q", pre) 348 | }, 349 | "join": func(sep string, ins ...string) string { 350 | return strings.Join(ins, sep) 351 | }, 352 | } 353 | 354 | func (m *memoryStore) buildResourceWithIndex(gvr store.GroupVersionResource, cluster string, obj interface{}) (string, string, store.Object) { 355 | s := store.Object{ 356 | Index: map[string]string{}, 357 | Obj: obj, 358 | } 359 | mobj := utils.Obj2JSONMap(obj) 360 | jp := jsonpath.New("parser") 361 | jp.AllowMissingKeys(true) 362 | gotmpl := template.New("parser").Funcs(funMap) 363 | for k, v := range m.indexConf[gvr] { 364 | w := bytes.NewBuffer([]byte{}) 365 | var exec interface { 366 | Execute(wr io.Writer, data interface{}) error 367 | } 368 | var err error 369 | if strings.Contains(v, "{{") { 370 | // go template 371 | exec, err = gotmpl.Parse(v) 372 | } else if !strings.Contains(v, "{") { 373 | // raw string 374 | s.Index[k] = v 375 | continue 376 | } else { 377 | // json path 378 | _ = jp.Parse(v) 379 | exec = jp 380 | } 381 | if err != nil { 382 | log.Errorf("parse temp error: %v", err) 383 | s.Index[k] = w.String() 384 | continue 385 | } 386 | err = exec.Execute(w, mobj) 387 | if err != nil { 388 | log.Warnf("exec jsonpath error: %v, %v", obj, err) 389 | } 390 | s.Index[k] = w.String() 391 | } 392 | namespace := "" 393 | name := "" 394 | if ns, ok := s.Index["namespace"]; ok { 395 | namespace = ns 396 | } 397 | if n, ok := s.Index["name"]; ok { 398 | name = n 399 | } 400 | s.Index["cluster"] = cluster 401 | if oo, ok := obj.(v1.Object); ok { 402 | // BUILD-IN Index: deletion 403 | if oo.GetDeletionTimestamp() != nil { 404 | s.Index["is_deleted"] = "true" 405 | } else { 406 | s.Index["is_deleted"] = "false" 407 | } 408 | if len(oo.GetAnnotations()) == 0 { 409 | oo.SetAnnotations(map[string]string{ 410 | constants.DSMClusterAnno: cluster, 411 | }) 412 | } else { 413 | anno := oo.GetAnnotations() 414 | anno[constants.DSMClusterAnno] = cluster 415 | oo.SetAnnotations(anno) 416 | } 417 | anno := oo.GetAnnotations() 418 | index, _ := json.Marshal(s.Index) 419 | anno[constants.IndexAnno] = string(index) // todo constants 420 | oo.SetAnnotations(anno) 421 | s.Obj = oo 422 | namespace = oo.GetNamespace() 423 | name = oo.GetName() 424 | s.Index["namespace"] = namespace 425 | s.Index["name"] = name 426 | } 427 | log.Debugf("memory store: gvr: %v, resources %s/%s, index: %v", gvr, namespace, name, s.Index) 428 | return namespace, name, s 429 | } 430 | -------------------------------------------------------------------------------- /store/memory/memory_test.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | corev1 "k8s.io/api/core/v1" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/runtime" 13 | 14 | "github.com/DaoCloud/ckube/common/constants" 15 | "github.com/DaoCloud/ckube/page" 16 | "github.com/DaoCloud/ckube/store" 17 | ) 18 | 19 | var podsGVR = store.GroupVersionResource{ 20 | Group: "", 21 | Version: "corev1", 22 | Resource: "pods", 23 | } 24 | 25 | var depsGVR = store.GroupVersionResource{ 26 | Group: "apps", 27 | Version: "corev1", 28 | Resource: "deployments", 29 | } 30 | 31 | func TestConcurrentReadWrite(t *testing.T) { 32 | m := NewMemoryStore(map[store.GroupVersionResource]map[string]string{ 33 | podsGVR: { 34 | "name": "{.metadata.name}", 35 | }, 36 | }) 37 | wg := sync.WaitGroup{} 38 | round := 1000 39 | wg.Add(round) 40 | go func() { 41 | for i := 0; i < round; i++ { 42 | _ = m.OnResourceAdded(podsGVR, "test", &corev1.Pod{ 43 | ObjectMeta: metav1.ObjectMeta{ 44 | Name: fmt.Sprintf("test-%d", i), 45 | Namespace: "test", 46 | }, 47 | }) 48 | wg.Done() 49 | } 50 | }() 51 | wg.Add(round) 52 | go func() { 53 | for i := 0; i < round; i++ { 54 | _ = m.OnResourceDeleted(podsGVR, "test", &corev1.Pod{ 55 | ObjectMeta: metav1.ObjectMeta{ 56 | Name: fmt.Sprintf("test-%d", i), 57 | Namespace: "test", 58 | }, 59 | }) 60 | wg.Done() 61 | } 62 | }() 63 | wg.Add(round) 64 | go func() { 65 | for i := 0; i < round; i++ { 66 | m.Get(podsGVR, "test", "test", fmt.Sprintf("test-%d", i)) 67 | wg.Done() 68 | } 69 | }() 70 | wg.Wait() 71 | } 72 | 73 | func BenchmarkWrite(b *testing.B) { 74 | m := NewMemoryStore(map[store.GroupVersionResource]map[string]string{ 75 | podsGVR: { 76 | "name": "{.metadata.name}", 77 | }, 78 | }) 79 | for i := 0; i < b.N; i++ { 80 | _ = m.OnResourceAdded(podsGVR, "test", &corev1.Pod{ 81 | ObjectMeta: metav1.ObjectMeta{ 82 | Name: fmt.Sprintf("test-%d", i), 83 | Namespace: "test", 84 | }, 85 | }) 86 | } 87 | } 88 | 89 | func BenchmarkRead(b *testing.B) { 90 | m := NewMemoryStore(map[store.GroupVersionResource]map[string]string{ 91 | podsGVR: { 92 | "name": "{.metadata.name}", 93 | }, 94 | }) 95 | go func() { 96 | for i := 0; i < b.N; i++ { 97 | _ = m.OnResourceAdded(podsGVR, "test", &corev1.Pod{ 98 | ObjectMeta: metav1.ObjectMeta{ 99 | Name: fmt.Sprintf("test-%d", i), 100 | Namespace: "test", 101 | }, 102 | }) 103 | } 104 | }() 105 | for i := 0; i < b.N; i++ { 106 | m.Get(podsGVR, "test", "test", fmt.Sprintf("test-%d", i)) 107 | } 108 | } 109 | 110 | var testIndexConf = map[store.GroupVersionResource]map[string]string{ 111 | podsGVR: { 112 | "namespace": "{.metadata.namespace}", 113 | "name": "{.metadata.name}", 114 | "uid": "{.metadata.uid}", 115 | }, 116 | depsGVR: { 117 | "namespace": "{.metadata.namespace}", 118 | "name": "{.metadata.name}", 119 | "uid": "{.metadata.uid}", 120 | "replicas": "{.spec.replicas}", 121 | }, 122 | } 123 | 124 | func TestMemoryStore_Query(t *testing.T) { 125 | cases := []struct { 126 | name string 127 | gvr store.GroupVersionResource 128 | resources []runtime.Object 129 | query store.Query 130 | res store.QueryResult 131 | }{ 132 | { 133 | name: "page & pagesize", 134 | gvr: podsGVR, 135 | resources: append([]runtime.Object{}, &corev1.Pod{ 136 | ObjectMeta: metav1.ObjectMeta{ 137 | Name: "test1", 138 | Namespace: "test", 139 | UID: "test", 140 | }, 141 | }, &corev1.Pod{ 142 | ObjectMeta: metav1.ObjectMeta{ 143 | Name: "test2", 144 | Namespace: "test", 145 | UID: "test", 146 | }, 147 | }), 148 | query: store.Query{ 149 | Namespace: "test", 150 | Paginate: page.Paginate{ 151 | Page: 1, 152 | PageSize: 1, 153 | }, 154 | }, 155 | res: store.QueryResult{ 156 | Error: nil, 157 | Items: append([]interface{}{}, &corev1.Pod{ 158 | ObjectMeta: metav1.ObjectMeta{ 159 | Name: "test1", 160 | Namespace: "test", 161 | UID: "test", 162 | Annotations: map[string]string{ 163 | constants.DSMClusterAnno: "", 164 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test1\",\"namespace\":\"test\",\"uid\":\"test\"}", 165 | }, 166 | }, 167 | }), 168 | Total: 2, 169 | }, 170 | }, 171 | { 172 | name: "page & pagesize 2", 173 | gvr: podsGVR, 174 | resources: append([]runtime.Object{}, &corev1.Pod{ 175 | ObjectMeta: metav1.ObjectMeta{ 176 | Name: "test1", 177 | Namespace: "test", 178 | UID: "test", 179 | }, 180 | }, &corev1.Pod{ 181 | ObjectMeta: metav1.ObjectMeta{ 182 | Name: "test2", 183 | Namespace: "test", 184 | UID: "test", 185 | }, 186 | }), 187 | query: store.Query{ 188 | Namespace: "test", 189 | Paginate: page.Paginate{ 190 | Page: 2, 191 | PageSize: 1, 192 | }, 193 | }, 194 | res: store.QueryResult{ 195 | Error: nil, 196 | Items: append([]interface{}{}, &corev1.Pod{ 197 | ObjectMeta: metav1.ObjectMeta{ 198 | Name: "test2", 199 | Namespace: "test", 200 | UID: "test", 201 | Annotations: map[string]string{ 202 | constants.DSMClusterAnno: "", 203 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test2\",\"namespace\":\"test\",\"uid\":\"test\"}", 204 | }, 205 | }, 206 | }), 207 | Total: 2, 208 | }, 209 | }, 210 | { 211 | name: "fuzzy search", 212 | gvr: podsGVR, 213 | resources: append([]runtime.Object{}, &corev1.Pod{ 214 | ObjectMeta: metav1.ObjectMeta{ 215 | Name: "hello", 216 | Namespace: "test", 217 | UID: "test", 218 | }, 219 | }, &corev1.Pod{ 220 | ObjectMeta: metav1.ObjectMeta{ 221 | Name: "llo", 222 | Namespace: "test", 223 | UID: "test", 224 | }, 225 | }, &corev1.Pod{ 226 | ObjectMeta: metav1.ObjectMeta{ 227 | Name: "l1lo", 228 | Namespace: "test", 229 | UID: "test", 230 | }, 231 | }), 232 | query: store.Query{ 233 | Namespace: "test", 234 | Paginate: page.Paginate{ 235 | Search: "llo", 236 | }, 237 | }, 238 | res: store.QueryResult{ 239 | Error: nil, 240 | Items: append([]interface{}{}, &corev1.Pod{ 241 | ObjectMeta: metav1.ObjectMeta{ 242 | Name: "hello", 243 | Namespace: "test", 244 | UID: "test", 245 | Annotations: map[string]string{ 246 | constants.DSMClusterAnno: "", 247 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"hello\",\"namespace\":\"test\",\"uid\":\"test\"}", 248 | }, 249 | }, 250 | }, &corev1.Pod{ 251 | ObjectMeta: metav1.ObjectMeta{ 252 | Name: "llo", 253 | Namespace: "test", 254 | UID: "test", 255 | Annotations: map[string]string{ 256 | constants.DSMClusterAnno: "", 257 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"llo\",\"namespace\":\"test\",\"uid\":\"test\"}", 258 | }, 259 | }, 260 | }), 261 | Total: 2, 262 | }, 263 | }, 264 | { 265 | name: "full search", 266 | gvr: podsGVR, 267 | resources: append([]runtime.Object{}, &corev1.Pod{ 268 | ObjectMeta: metav1.ObjectMeta{ 269 | Name: "hello", 270 | Namespace: "test", 271 | UID: "test", 272 | }, 273 | }, &corev1.Pod{ 274 | ObjectMeta: metav1.ObjectMeta{ 275 | Name: "llo", 276 | Namespace: "test", 277 | UID: "test", 278 | }, 279 | }, &corev1.Pod{ 280 | ObjectMeta: metav1.ObjectMeta{ 281 | Name: "l1lo", 282 | Namespace: "test", 283 | UID: "test", 284 | }, 285 | }), 286 | query: store.Query{ 287 | Namespace: "test", 288 | Paginate: page.Paginate{ 289 | Search: "name=\"llo\"", 290 | }, 291 | }, 292 | res: store.QueryResult{ 293 | Error: nil, 294 | Items: append([]interface{}{}, &corev1.Pod{ 295 | ObjectMeta: metav1.ObjectMeta{ 296 | Name: "llo", 297 | Namespace: "test", 298 | UID: "test", 299 | Annotations: map[string]string{ 300 | constants.DSMClusterAnno: "", 301 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"llo\",\"namespace\":\"test\",\"uid\":\"test\"}", 302 | }, 303 | }, 304 | }), 305 | Total: 1, 306 | }, 307 | }, 308 | { 309 | name: "search key missing", 310 | gvr: podsGVR, 311 | resources: append([]runtime.Object{}, &corev1.Pod{ 312 | ObjectMeta: metav1.ObjectMeta{ 313 | Name: "hello", 314 | Namespace: "test", 315 | UID: "test", 316 | }, 317 | }, &corev1.Pod{ 318 | ObjectMeta: metav1.ObjectMeta{ 319 | Name: "llo", 320 | Namespace: "test", 321 | UID: "test", 322 | }, 323 | }, &corev1.Pod{ 324 | ObjectMeta: metav1.ObjectMeta{ 325 | Name: "l1lo", 326 | Namespace: "test", 327 | UID: "test", 328 | }, 329 | }), 330 | query: store.Query{ 331 | Namespace: "test", 332 | Paginate: page.Paginate{ 333 | Search: "name1=llo", 334 | }, 335 | }, 336 | res: store.QueryResult{ 337 | Error: fmt.Errorf("unexpected search key: name1"), 338 | Total: 0, 339 | }, 340 | }, 341 | { 342 | name: "advance search", 343 | gvr: podsGVR, 344 | resources: append([]runtime.Object{}, &corev1.Pod{ 345 | ObjectMeta: metav1.ObjectMeta{ 346 | Name: "hello", 347 | Namespace: "test", 348 | UID: "test", 349 | }, 350 | }, &corev1.Pod{ 351 | ObjectMeta: metav1.ObjectMeta{ 352 | Name: "llo", 353 | Namespace: "test", 354 | UID: "test", 355 | }, 356 | }, &corev1.Pod{ 357 | ObjectMeta: metav1.ObjectMeta{ 358 | Name: "l1lo", 359 | Namespace: "test", 360 | UID: "test", 361 | }, 362 | }), 363 | query: store.Query{ 364 | Namespace: "test", 365 | Paginate: page.Paginate{ 366 | Search: "__ckube_as__:name in (hello, l1lo)", 367 | }, 368 | }, 369 | res: store.QueryResult{ 370 | Error: nil, 371 | Items: append([]interface{}{}, &corev1.Pod{ 372 | ObjectMeta: metav1.ObjectMeta{ 373 | Name: "hello", 374 | Namespace: "test", 375 | UID: "test", 376 | Annotations: map[string]string{ 377 | constants.DSMClusterAnno: "", 378 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"hello\",\"namespace\":\"test\",\"uid\":\"test\"}", 379 | }, 380 | }, 381 | }, &corev1.Pod{ 382 | ObjectMeta: metav1.ObjectMeta{ 383 | Name: "l1lo", 384 | Namespace: "test", 385 | UID: "test", 386 | Annotations: map[string]string{ 387 | constants.DSMClusterAnno: "", 388 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"l1lo\",\"namespace\":\"test\",\"uid\":\"test\"}", 389 | }, 390 | }, 391 | }), 392 | Total: 2, 393 | }, 394 | }, 395 | { 396 | name: "advance search key missing", 397 | gvr: podsGVR, 398 | resources: append([]runtime.Object{}, &corev1.Pod{ 399 | ObjectMeta: metav1.ObjectMeta{ 400 | Name: "hello", 401 | Namespace: "test", 402 | UID: "test", 403 | }, 404 | }, &corev1.Pod{ 405 | ObjectMeta: metav1.ObjectMeta{ 406 | Name: "llo", 407 | Namespace: "test", 408 | UID: "test", 409 | }, 410 | }, &corev1.Pod{ 411 | ObjectMeta: metav1.ObjectMeta{ 412 | Name: "l1lo", 413 | Namespace: "test", 414 | UID: "test", 415 | }, 416 | }), 417 | query: store.Query{ 418 | Namespace: "test", 419 | Paginate: page.Paginate{ 420 | Search: "__ckube_as__:test in (hello, l1lo)", 421 | }, 422 | }, 423 | res: store.QueryResult{ 424 | Error: nil, 425 | Total: 0, 426 | }, 427 | }, 428 | { 429 | name: "advance search format error", 430 | gvr: podsGVR, 431 | resources: append([]runtime.Object{}, &corev1.Pod{ 432 | ObjectMeta: metav1.ObjectMeta{ 433 | Name: "hello", 434 | Namespace: "test", 435 | UID: "test", 436 | }, 437 | }), 438 | query: store.Query{ 439 | Namespace: "test", 440 | Paginate: page.Paginate{ 441 | Search: "__ckube_as__:test xxx", 442 | }, 443 | }, 444 | res: store.QueryResult{ 445 | Error: fmt.Errorf("couldn't parse the selector string \"test xxx\": unable to parse requirement: found 'xxx', expected: '=', '!=', '==', 'in', notin'"), 446 | Total: 0, 447 | }, 448 | }, 449 | { 450 | name: "sort single key", 451 | gvr: podsGVR, 452 | resources: append([]runtime.Object{}, &corev1.Pod{ 453 | ObjectMeta: metav1.ObjectMeta{ 454 | Name: "test1", 455 | Namespace: "test", 456 | UID: "2", 457 | }, 458 | }, &corev1.Pod{ 459 | ObjectMeta: metav1.ObjectMeta{ 460 | Name: "test5", 461 | Namespace: "test", 462 | UID: "1", 463 | }, 464 | }, &corev1.Pod{ 465 | ObjectMeta: metav1.ObjectMeta{ 466 | Name: "test3", 467 | Namespace: "test", 468 | UID: "3", 469 | }, 470 | }), 471 | query: store.Query{ 472 | Namespace: "test", 473 | Paginate: page.Paginate{ 474 | Sort: "uid!str", 475 | }, 476 | }, 477 | res: store.QueryResult{ 478 | Error: nil, 479 | Items: append([]interface{}{}, &corev1.Pod{ 480 | ObjectMeta: metav1.ObjectMeta{ 481 | Name: "test5", 482 | Namespace: "test", 483 | UID: "1", 484 | Annotations: map[string]string{ 485 | constants.DSMClusterAnno: "", 486 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test5\",\"namespace\":\"test\",\"uid\":\"1\"}", 487 | }, 488 | }, 489 | }, &corev1.Pod{ 490 | ObjectMeta: metav1.ObjectMeta{ 491 | Name: "test1", 492 | Namespace: "test", 493 | UID: "2", 494 | Annotations: map[string]string{ 495 | constants.DSMClusterAnno: "", 496 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test1\",\"namespace\":\"test\",\"uid\":\"2\"}", 497 | }, 498 | }, 499 | }, &corev1.Pod{ 500 | ObjectMeta: metav1.ObjectMeta{ 501 | Name: "test3", 502 | Namespace: "test", 503 | UID: "3", 504 | Annotations: map[string]string{ 505 | constants.DSMClusterAnno: "", 506 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test3\",\"namespace\":\"test\",\"uid\":\"3\"}", 507 | }, 508 | }, 509 | }), 510 | Total: 3, 511 | }, 512 | }, 513 | { 514 | name: "sort single key desc", 515 | gvr: podsGVR, 516 | resources: append([]runtime.Object{}, &corev1.Pod{ 517 | ObjectMeta: metav1.ObjectMeta{ 518 | Name: "test1", 519 | Namespace: "test", 520 | UID: "2", 521 | }, 522 | }, &corev1.Pod{ 523 | ObjectMeta: metav1.ObjectMeta{ 524 | Name: "test5", 525 | Namespace: "test", 526 | UID: "1", 527 | }, 528 | }, &corev1.Pod{ 529 | ObjectMeta: metav1.ObjectMeta{ 530 | Name: "test3", 531 | Namespace: "test", 532 | UID: "3", 533 | }, 534 | }), 535 | query: store.Query{ 536 | Namespace: "test", 537 | Paginate: page.Paginate{ 538 | Sort: "uid desc", 539 | }, 540 | }, 541 | res: store.QueryResult{ 542 | Error: nil, 543 | Items: append([]interface{}{}, &corev1.Pod{ 544 | ObjectMeta: metav1.ObjectMeta{ 545 | Name: "test3", 546 | Namespace: "test", 547 | UID: "3", 548 | Annotations: map[string]string{ 549 | constants.DSMClusterAnno: "", 550 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test3\",\"namespace\":\"test\",\"uid\":\"3\"}", 551 | }, 552 | }, 553 | }, &corev1.Pod{ 554 | ObjectMeta: metav1.ObjectMeta{ 555 | Name: "test1", 556 | Namespace: "test", 557 | UID: "2", 558 | Annotations: map[string]string{ 559 | constants.DSMClusterAnno: "", 560 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test1\",\"namespace\":\"test\",\"uid\":\"2\"}", 561 | }, 562 | }, 563 | }, &corev1.Pod{ 564 | ObjectMeta: metav1.ObjectMeta{ 565 | Name: "test5", 566 | Namespace: "test", 567 | UID: "1", 568 | Annotations: map[string]string{ 569 | constants.DSMClusterAnno: "", 570 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test5\",\"namespace\":\"test\",\"uid\":\"1\"}", 571 | }, 572 | }, 573 | }), 574 | Total: 3, 575 | }, 576 | }, 577 | { 578 | name: "sort invalid key convert", 579 | gvr: podsGVR, 580 | resources: append([]runtime.Object{}, &corev1.Pod{ 581 | ObjectMeta: metav1.ObjectMeta{ 582 | Name: "test1", 583 | Namespace: "test", 584 | UID: "2", 585 | }, 586 | }, &corev1.Pod{ 587 | ObjectMeta: metav1.ObjectMeta{ 588 | Name: "test5", 589 | Namespace: "test", 590 | UID: "1", 591 | }, 592 | }), 593 | query: store.Query{ 594 | Namespace: "test", 595 | Paginate: page.Paginate{ 596 | Sort: "name!int", 597 | }, 598 | }, 599 | res: store.QueryResult{ 600 | Error: fmt.Errorf("value of `name` can not convert to number"), 601 | Total: 0, 602 | }, 603 | }, 604 | { 605 | name: "multiple keys desc", 606 | gvr: podsGVR, 607 | resources: append([]runtime.Object{}, &corev1.Pod{ 608 | ObjectMeta: metav1.ObjectMeta{ 609 | Name: "test1", 610 | Namespace: "test", 611 | UID: "2", 612 | }, 613 | }, &corev1.Pod{ 614 | ObjectMeta: metav1.ObjectMeta{ 615 | Name: "test5", 616 | Namespace: "test1", 617 | UID: "1", 618 | }, 619 | }, &corev1.Pod{ 620 | ObjectMeta: metav1.ObjectMeta{ 621 | Name: "test3", 622 | Namespace: "test", 623 | UID: "3", 624 | }, 625 | }), 626 | query: store.Query{ 627 | Namespace: "", 628 | Paginate: page.Paginate{ 629 | Sort: "namespace,uid desc", 630 | }, 631 | }, 632 | res: store.QueryResult{ 633 | Error: nil, 634 | Items: append([]interface{}{}, &corev1.Pod{ 635 | ObjectMeta: metav1.ObjectMeta{ 636 | Name: "test3", 637 | Namespace: "test", 638 | UID: "3", 639 | Annotations: map[string]string{ 640 | constants.DSMClusterAnno: "", 641 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test3\",\"namespace\":\"test\",\"uid\":\"3\"}", 642 | }, 643 | }, 644 | }, &corev1.Pod{ 645 | ObjectMeta: metav1.ObjectMeta{ 646 | Name: "test1", 647 | Namespace: "test", 648 | UID: "2", 649 | Annotations: map[string]string{ 650 | constants.DSMClusterAnno: "", 651 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test1\",\"namespace\":\"test\",\"uid\":\"2\"}", 652 | }, 653 | }, 654 | }, &corev1.Pod{ 655 | ObjectMeta: metav1.ObjectMeta{ 656 | Name: "test5", 657 | Namespace: "test1", 658 | UID: "1", 659 | Annotations: map[string]string{ 660 | constants.DSMClusterAnno: "", 661 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test5\",\"namespace\":\"test1\",\"uid\":\"1\"}", 662 | }, 663 | }, 664 | }), 665 | Total: 3, 666 | }, 667 | }, 668 | { 669 | name: "sort type", 670 | gvr: podsGVR, 671 | resources: append([]runtime.Object{}, &corev1.Pod{ 672 | ObjectMeta: metav1.ObjectMeta{ 673 | Name: "test1", 674 | Namespace: "test", 675 | UID: "11", 676 | }, 677 | }, &corev1.Pod{ 678 | ObjectMeta: metav1.ObjectMeta{ 679 | Name: "test5", 680 | Namespace: "test", 681 | UID: "2", 682 | }, 683 | }, &corev1.Pod{ 684 | ObjectMeta: metav1.ObjectMeta{ 685 | Name: "test3", 686 | Namespace: "test", 687 | UID: "3", 688 | }, 689 | }), 690 | query: store.Query{ 691 | Namespace: "", 692 | Paginate: page.Paginate{ 693 | Sort: "uid!int", 694 | }, 695 | }, 696 | res: store.QueryResult{ 697 | Error: nil, 698 | Items: append([]interface{}{}, &corev1.Pod{ 699 | ObjectMeta: metav1.ObjectMeta{ 700 | Name: "test5", 701 | Namespace: "test", 702 | UID: "2", 703 | Annotations: map[string]string{ 704 | constants.DSMClusterAnno: "", 705 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test5\",\"namespace\":\"test\",\"uid\":\"2\"}", 706 | }, 707 | }, 708 | }, &corev1.Pod{ 709 | ObjectMeta: metav1.ObjectMeta{ 710 | Name: "test3", 711 | Namespace: "test", 712 | UID: "3", 713 | Annotations: map[string]string{ 714 | constants.DSMClusterAnno: "", 715 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test3\",\"namespace\":\"test\",\"uid\":\"3\"}", 716 | }, 717 | }, 718 | }, &corev1.Pod{ 719 | ObjectMeta: metav1.ObjectMeta{ 720 | Name: "test1", 721 | Namespace: "test", 722 | UID: "11", 723 | Annotations: map[string]string{ 724 | constants.DSMClusterAnno: "", 725 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test1\",\"namespace\":\"test\",\"uid\":\"11\"}", 726 | }, 727 | }, 728 | }), 729 | Total: 3, 730 | }, 731 | }, 732 | { 733 | name: "union search", 734 | gvr: podsGVR, 735 | resources: append([]runtime.Object{}, &corev1.Pod{ 736 | ObjectMeta: metav1.ObjectMeta{ 737 | Name: "test1", 738 | Namespace: "test", 739 | UID: "11", 740 | }, 741 | }, &corev1.Pod{ 742 | ObjectMeta: metav1.ObjectMeta{ 743 | Name: "ok", 744 | Namespace: "test", 745 | UID: "2", 746 | }, 747 | }, &corev1.Pod{ 748 | ObjectMeta: metav1.ObjectMeta{ 749 | Name: "0tes", 750 | Namespace: "test", 751 | UID: "3", 752 | }, 753 | }, &corev1.Pod{ 754 | ObjectMeta: metav1.ObjectMeta{ 755 | Name: "test3", 756 | Namespace: "test", 757 | UID: "3", 758 | }, 759 | }, &corev1.Pod{ 760 | ObjectMeta: metav1.ObjectMeta{ 761 | Name: "test3", 762 | Namespace: "test1", 763 | UID: "30", 764 | }, 765 | }, &corev1.Pod{ 766 | ObjectMeta: metav1.ObjectMeta{ 767 | Name: "test13", 768 | Namespace: "test1", 769 | UID: "20", 770 | }, 771 | }, &corev1.Pod{ 772 | ObjectMeta: metav1.ObjectMeta{ 773 | Name: "ok", 774 | Namespace: "test1", 775 | UID: "2", 776 | }, 777 | }), 778 | query: store.Query{ 779 | Namespace: "", 780 | Paginate: page.Paginate{ 781 | Page: 1, 782 | PageSize: 3, 783 | Search: "name=test; __ckube_as__:name notin (ok)", 784 | Sort: "namespace,uid!int", 785 | }, 786 | }, 787 | res: store.QueryResult{ 788 | Error: nil, 789 | Items: append([]interface{}{}, &corev1.Pod{ 790 | ObjectMeta: metav1.ObjectMeta{ 791 | Name: "test3", 792 | Namespace: "test", 793 | UID: "3", 794 | Annotations: map[string]string{ 795 | constants.DSMClusterAnno: "", 796 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test3\",\"namespace\":\"test\",\"uid\":\"3\"}", 797 | }, 798 | }, 799 | }, &corev1.Pod{ 800 | ObjectMeta: metav1.ObjectMeta{ 801 | Name: "test1", 802 | Namespace: "test", 803 | UID: "11", 804 | Annotations: map[string]string{ 805 | constants.DSMClusterAnno: "", 806 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test1\",\"namespace\":\"test\",\"uid\":\"11\"}", 807 | }, 808 | }, 809 | }, &corev1.Pod{ 810 | ObjectMeta: metav1.ObjectMeta{ 811 | Name: "test13", 812 | Namespace: "test1", 813 | UID: "20", 814 | Annotations: map[string]string{ 815 | constants.DSMClusterAnno: "", 816 | constants.IndexAnno: "{\"cluster\":\"\",\"is_deleted\":\"false\",\"name\":\"test13\",\"namespace\":\"test1\",\"uid\":\"20\"}", 817 | }, 818 | }, 819 | }), 820 | Total: 4, 821 | }, 822 | }, 823 | } 824 | for i, c := range cases { 825 | t.Run(fmt.Sprintf("%d-%s", i, c.name), func(t *testing.T) { 826 | s := NewMemoryStore(testIndexConf) 827 | for _, r := range c.resources { 828 | _ = s.OnResourceAdded(c.gvr, "", r) 829 | } 830 | res := s.Query(c.gvr, c.query) 831 | assert.Equal(t, c.res, res) 832 | }) 833 | } 834 | } 835 | 836 | func TestMemoryStore_buildResourceWithIndex(t *testing.T) { 837 | cases := []struct { 838 | name string 839 | index map[string]string 840 | obj interface{} 841 | expectedIndex map[string]string 842 | }{ 843 | { 844 | name: "jsonpath", 845 | index: map[string]string{ 846 | "namespace": "{.metadata.namespace}", 847 | "name": "{.metadata.name}", 848 | "containers": "{.spec.containers[*].name}", 849 | "status": "{.status.phase}", 850 | }, 851 | obj: &corev1.Pod{ 852 | ObjectMeta: metav1.ObjectMeta{ 853 | Name: "test-1", 854 | Namespace: "default", 855 | }, 856 | Spec: corev1.PodSpec{ 857 | Containers: []corev1.Container{ 858 | {Name: "c1"}, 859 | {Name: "c2"}, 860 | }, 861 | }, 862 | Status: corev1.PodStatus{ 863 | Phase: corev1.PodRunning, 864 | }, 865 | }, 866 | expectedIndex: map[string]string{ 867 | "namespace": "default", 868 | "name": "test-1", 869 | "containers": "c1 c2", 870 | "status": "Running", 871 | }, 872 | }, 873 | { 874 | name: "go tmpl", 875 | index: map[string]string{ 876 | "status": `{{if .metadata.deletionTimestamp }}Deleting{{else}}{{.status.phase}}{{end}}`, 877 | "default status": `{{ .x | default "no spec"}}`, 878 | "quote": `{{ .status.phase | quote }}`, 879 | "join": `{{ join "/" .metadata.namespace .metadata.name }}`, 880 | "raw": "test raw", 881 | }, 882 | obj: &corev1.Pod{ 883 | ObjectMeta: metav1.ObjectMeta{ 884 | Name: "test-1", 885 | Namespace: "default", 886 | DeletionTimestamp: &metav1.Time{ 887 | Time: time.Now(), 888 | }, 889 | }, 890 | Status: corev1.PodStatus{ 891 | Phase: corev1.PodRunning, 892 | }, 893 | }, 894 | expectedIndex: map[string]string{ 895 | "join": "default/test-1", 896 | "status": "Deleting", 897 | "default status": "no spec", 898 | "quote": "\"Running\"", 899 | "raw": "test raw", 900 | "name": "test-1", 901 | "namespace": "default", 902 | }, 903 | }, 904 | } 905 | for _, c := range cases { 906 | t.Run(c.name, func(t *testing.T) { 907 | gvr := store.GroupVersionResource{} 908 | m := memoryStore{ 909 | indexConf: map[store.GroupVersionResource]map[string]string{ 910 | gvr: c.index, 911 | }, 912 | } 913 | _, _, o := m.buildResourceWithIndex(gvr, "test", c.obj) 914 | delete(o.Index, "is_deleted") 915 | delete(o.Index, "cluster") 916 | assert.Equal(t, c.expectedIndex, o.Index) 917 | }) 918 | } 919 | } 920 | -------------------------------------------------------------------------------- /store/model.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | type GroupVersionResource struct { 4 | Group string 5 | Version string 6 | Resource string 7 | } 8 | 9 | type QueryResult struct { 10 | Error error `json:"error,omitempty"` 11 | Items []interface{} `json:"items"` 12 | Total int64 `json:"total"` 13 | } 14 | 15 | type Object struct { 16 | Index map[string]string 17 | Obj interface{} 18 | } 19 | -------------------------------------------------------------------------------- /utils/fs_watcher.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "time" 7 | 8 | "github.com/fsnotify/fsnotify" 9 | 10 | "github.com/DaoCloud/ckube/log" 11 | ) 12 | 13 | type FixedFileWatcher interface { 14 | io.Closer 15 | Start() error 16 | Events() <-chan Event 17 | } 18 | 19 | type Event struct { 20 | Name string 21 | Type EventType 22 | } 23 | 24 | type EventType int 25 | 26 | const ( 27 | EventTypeChanged = iota 28 | EventTypeError 29 | ) 30 | 31 | type fixedFileWatcher struct { 32 | files []string 33 | fswatcher *fsnotify.Watcher 34 | events chan Event 35 | } 36 | 37 | func NewFixedFileWatcher(files []string) (FixedFileWatcher, error) { 38 | w, err := fsnotify.NewWatcher() 39 | if err != nil { 40 | return nil, err 41 | } 42 | for _, f := range files { 43 | if _, err := os.Stat(f); err != nil { 44 | return nil, err 45 | } 46 | if err := w.Add(f); err != nil { 47 | return nil, err 48 | } 49 | } 50 | return &fixedFileWatcher{ 51 | files: files, 52 | events: make(chan Event), 53 | }, nil 54 | } 55 | 56 | func (w *fixedFileWatcher) Start() error { 57 | ww, err := fsnotify.NewWatcher() 58 | if err != nil { 59 | return err 60 | } 61 | w.fswatcher = ww 62 | for _, f := range w.files { 63 | if err := w.fswatcher.Add(f); err != nil { 64 | return err 65 | } 66 | } 67 | go func() { 68 | for { // nolint:gosimple 69 | select { 70 | case e, open := <-w.fswatcher.Events: 71 | if !open { 72 | log.Info("fs watcher closed") 73 | return 74 | } 75 | log.Infof("get file watcher event: %v", e) 76 | switch e.Op { 77 | case fsnotify.Write: 78 | // do reload 79 | case fsnotify.Remove: 80 | // 在 Kubernetes 里面，当挂载 ConfigMap 的时候，如果发生文件重新，Kubernetes 会首先删除这个文件 81 | // 再重新创建，所以我们应该在删除之后重新建立 watcher。 82 | _ = w.fswatcher.Remove(e.Name) 83 | time.Sleep(time.Second * 2) 84 | // 等待一定时间之后重新加入 watcher 队列 85 | err := w.fswatcher.Add(e.Name) 86 | if err != nil { 87 | log.Errorf("add watcher for %s error: %v", e.Name, err) 88 | w.events <- Event{ 89 | Name: e.Name, 90 | Type: EventTypeError, 91 | } 92 | continue 93 | } 94 | // do reload 95 | default: 96 | // do not reload 97 | continue 98 | } 99 | w.events <- Event{ 100 | Name: e.Name, 101 | Type: EventTypeChanged, 102 | } 103 | } 104 | } 105 | }() 106 | return nil 107 | } 108 | 109 | func (w *fixedFileWatcher) Close() error { 110 | if w.fswatcher != nil { 111 | w.fswatcher.Close() 112 | w.fswatcher = nil 113 | close(w.events) 114 | w.events = nil 115 | } 116 | return nil 117 | } 118 | 119 | func (w *fixedFileWatcher) Events() <-chan Event { 120 | return w.events 121 | } 122 | -------------------------------------------------------------------------------- /utils/prommonitor/metrics.go: -------------------------------------------------------------------------------- 1 | package prommonitor 2 | 3 | import ( 4 | "github.com/DaoCloud/ckube/api" 5 | "github.com/prometheus/client_golang/prometheus" 6 | "github.com/prometheus/client_golang/prometheus/promauto" 7 | "github.com/prometheus/client_golang/prometheus/promhttp" 8 | ) 9 | 10 | const ( 11 | ComponentMetricsLabel string = "component" 12 | CkubeComponent string = "ckube" 13 | ) 14 | 15 | var ( 16 | Up = promauto.NewGaugeVec(prometheus.GaugeOpts{ 17 | Name: "up", 18 | Help: "Component up status", 19 | }, []string{ComponentMetricsLabel}) 20 | ConfigReload = promauto.NewCounterVec(prometheus.CounterOpts{ 21 | Name: "ckube_reload_config_total", 22 | Help: "Config reload count", 23 | }, []string{"status"}) 24 | Requests = promauto.NewCounterVec(prometheus.CounterOpts{ 25 | Name: "ckube_requests_total", 26 | Help: "Requests count", 27 | }, []string{"cluster", "group", "version", "kind", "single", "cached"}) 28 | Resources = promauto.NewGaugeVec(prometheus.GaugeOpts{ 29 | Name: "ckube_resources_total", 30 | Help: "resources count", 31 | }, []string{"cluster", "group", "version", "resource", "namespace"}) 32 | ) 33 | 34 | func PromHandler(r *api.ReqContext) interface{} { 35 | promhttp.Handler().ServeHTTP(r.Writer, r.Request) 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /utils/subset.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func IsSubsetOf(sub map[string]string, parent map[string]string) bool { 4 | for k, v := range sub { 5 | if vv, ok := parent[k]; !ok || vv != v { 6 | return false 7 | } 8 | } 9 | return true 10 | } 11 | -------------------------------------------------------------------------------- /utils/subset_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestIsSubsetOf(t *testing.T) { 10 | cases := []struct { 11 | name string 12 | sub map[string]string 13 | parent map[string]string 14 | expect bool 15 | }{ 16 | { 17 | "normal", 18 | map[string]string{ 19 | "a": "b", 20 | }, 21 | map[string]string{ 22 | "a": "b", 23 | "b": "c", 24 | }, 25 | true, 26 | }, 27 | { 28 | "multi sub", 29 | map[string]string{ 30 | "a": "b", 31 | "b": "c", 32 | }, 33 | map[string]string{ 34 | "a": "b", 35 | "b": "c", 36 | "c": "d", 37 | }, 38 | true, 39 | }, 40 | { 41 | "not sub", 42 | map[string]string{ 43 | "a": "c", 44 | }, 45 | map[string]string{ 46 | "a": "b", 47 | "b": "c", 48 | }, 49 | false, 50 | }, 51 | { 52 | "not sub 2", 53 | map[string]string{ 54 | "a": "c", 55 | }, 56 | map[string]string{ 57 | "b": "c", 58 | }, 59 | false, 60 | }, 61 | { 62 | "nil sub", 63 | nil, 64 | map[string]string{ 65 | "a": "b", 66 | "b": "c", 67 | }, 68 | true, 69 | }, 70 | { 71 | "nil parent", 72 | map[string]string{ 73 | "a": "c", 74 | }, 75 | nil, 76 | false, 77 | }, 78 | { 79 | "both nil", 80 | nil, 81 | nil, 82 | true, 83 | }, 84 | } 85 | for i, c := range cases { 86 | t.Run(fmt.Sprintf("%d---%s", i, c.name), func(t *testing.T) { 87 | res := IsSubsetOf(c.sub, c.parent) 88 | assert.Equal(t, c.expect, res) 89 | }) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "encoding/json" 4 | 5 | func Obj2JSONMap(obj interface{}) map[string]interface{} { 6 | m := make(map[string]interface{}) 7 | bs, _ := json.Marshal(obj) 8 | _ = json.Unmarshal(bs, &m) 9 | return m 10 | } 11 | -------------------------------------------------------------------------------- /watcher/interface.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | type Watcher interface { 4 | Start() error 5 | Stop() error 6 | } 7 | -------------------------------------------------------------------------------- /watcher/watcher.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/runtime" 13 | "k8s.io/apimachinery/pkg/runtime/schema" 14 | "k8s.io/apimachinery/pkg/watch" 15 | "k8s.io/client-go/kubernetes/scheme" 16 | "k8s.io/client-go/rest" 17 | 18 | "github.com/DaoCloud/ckube/common" 19 | "github.com/DaoCloud/ckube/log" 20 | "github.com/DaoCloud/ckube/store" 21 | ) 22 | 23 | type watcher struct { 24 | clusterConfigs map[string]rest.Config 25 | resources []store.GroupVersionResource 26 | store store.Store 27 | stop chan struct{} 28 | lock sync.Mutex 29 | Watcher 30 | } 31 | 32 | func NewWatcher(clusterConfigs map[string]rest.Config, resources []store.GroupVersionResource, store store.Store) Watcher { 33 | return &watcher{ 34 | clusterConfigs: clusterConfigs, 35 | resources: resources, 36 | store: store, 37 | stop: make(chan struct{}), 38 | } 39 | } 40 | 41 | func (w *watcher) Stop() error { 42 | close(w.stop) 43 | return nil 44 | } 45 | 46 | type ObjType struct { 47 | v1.TypeMeta `json:",inline"` 48 | v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 49 | Data map[string]interface{} 50 | } 51 | 52 | func (o *ObjType) UnmarshalJSON(bytes []byte) error { 53 | m := map[string]interface{}{} 54 | _ = json.Unmarshal(bytes, &m) 55 | if v, ok := m["apiVersion"]; ok { 56 | o.APIVersion = v.(string) 57 | } 58 | if v, ok := m["kind"]; ok { 59 | o.Kind = v.(string) 60 | } 61 | if meta, ok := m["metadata"]; ok { 62 | bs, _ := json.Marshal(meta) 63 | _ = json.Unmarshal(bs, &o.ObjectMeta) 64 | } 65 | delete(m, "apiVersion") 66 | delete(m, "kind") 67 | delete(m, "metadata") 68 | o.Data = m 69 | return nil 70 | } 71 | 72 | func (o *ObjType) MarshalJSON() ([]byte, error) { 73 | bsm, _ := json.Marshal(o.Data) 74 | bso, _ := json.Marshal(struct { 75 | v1.TypeMeta `json:",inline"` 76 | v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 77 | }{ 78 | TypeMeta: o.TypeMeta, 79 | ObjectMeta: o.ObjectMeta, 80 | }) 81 | if string(bsm) == "{}" || string(bsm) == "null" { 82 | return bso, nil 83 | } 84 | if string(bso) == "{}" { 85 | return bsm, nil 86 | } 87 | bsm = bsm[:len(bsm)-1] 88 | bso = bso[1:] 89 | bs := make([]byte, 0, len(bsm)+len(bso)+1) 90 | bs = append(bs, bsm...) 91 | bs = append(bs, ',') 92 | bs = append(bs, bso...) 93 | return bs, nil 94 | } 95 | 96 | func (o ObjType) GetObjectKind() schema.ObjectKind { 97 | return &o 98 | } 99 | 100 | func (o ObjType) DeepCopyObject() runtime.Object { 101 | // o.lock.Lock() 102 | // defer o.lock.Unlock() 103 | m := map[string]interface{}{} 104 | for k, v := range o.Data { 105 | m[k] = v 106 | } 107 | return &ObjType{ 108 | TypeMeta: o.TypeMeta, 109 | ObjectMeta: o.ObjectMeta, 110 | Data: m, 111 | } 112 | } 113 | 114 | func (w *watcher) watchResources(r store.GroupVersionResource, cluster string) { 115 | gvk := schema.GroupVersionKind{ 116 | Group: r.Group, 117 | Version: r.Version, 118 | Kind: strings.TrimRight(common.GetGVRKind(r.Group, r.Version, r.Resource), "List"), 119 | } 120 | gv := schema.GroupVersion{ 121 | Group: r.Group, 122 | Version: r.Version, 123 | } 124 | w.lock.Lock() 125 | if _, ok := scheme.Scheme.KnownTypes(gv)[gvk.Kind]; !ok { 126 | scheme.Scheme.AddKnownTypeWithName(gvk, &ObjType{}) 127 | } 128 | w.lock.Unlock() 129 | config := w.clusterConfigs[cluster] 130 | 131 | config.GroupVersion = &schema.GroupVersion{ 132 | Group: r.Group, 133 | Version: r.Version, 134 | } 135 | scheme.Codecs.UniversalDeserializer() 136 | config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 137 | rt, _ := rest.RESTClientFor(&config) 138 | for { 139 | select { 140 | case <-w.stop: 141 | return 142 | default: 143 | } 144 | ctx, cancel := context.WithTimeout(context.Background(), time.Hour) 145 | url := "" 146 | if r.Group == "" { 147 | url = fmt.Sprintf("/api/%s/%s?watch=true", r.Version, r.Resource) 148 | } else { 149 | url = fmt.Sprintf("/apis/%s/%s/%s?watch=true", r.Group, r.Version, r.Resource) 150 | } 151 | first := true 152 | ww, err := rt.Get().RequestURI(url).Timeout(time.Hour).Watch(ctx) 153 | if err != nil { 154 | log.Errorf("cluster(%s): create watcher for %s error: %v", cluster, url, err) 155 | time.Sleep(time.Second * 15) 156 | } else { 157 | resultChan: 158 | for { 159 | select { 160 | case rr, open := <-ww.ResultChan(): 161 | if first { 162 | // only clean resource at the first time 163 | // to avoid the resources gone after server break. 164 | _ = w.store.Clean(r, cluster) 165 | first = false 166 | } 167 | if open { 168 | switch rr.Type { 169 | case watch.Added: 170 | _ = w.store.OnResourceAdded(r, cluster, rr.Object) 171 | case watch.Modified: 172 | _ = w.store.OnResourceModified(r, cluster, rr.Object) 173 | case watch.Deleted: 174 | _ = w.store.OnResourceDeleted(r, cluster, rr.Object) 175 | case watch.Error: 176 | log.Warnf("cluster(%s): watch stream(%v) error: %v", cluster, r, rr.Object) 177 | } 178 | } else { 179 | log.Warnf("cluster(%s): watch stream(%v) closed", cluster, r) 180 | ww.Stop() 181 | time.Sleep(time.Second * 3) 182 | break resultChan 183 | } 184 | case <-w.stop: 185 | ww.Stop() 186 | cancel() 187 | return 188 | } 189 | } 190 | } 191 | cancel() 192 | } 193 | } 194 | 195 | func (w *watcher) Start() error { 196 | for _, r := range w.resources { 197 | for c := range w.clusterConfigs { 198 | go w.watchResources(r, c) 199 | } 200 | } 201 | return nil 202 | } 203 | --------------------------------------------------------------------------------