├── test ├── e2e │ ├── priorityclass.yaml │ ├── aliases.sh │ ├── delete-chaperon │ │ ├── test.yaml │ │ └── test.sh │ ├── values.yaml │ ├── delete-delegate │ │ ├── test.yaml │ │ └── test.sh │ ├── exec │ │ ├── test.yaml │ │ └── test.sh │ ├── logs │ │ ├── test.yaml │ │ └── test.sh │ ├── cleanup │ │ ├── test.yaml │ │ └── test.sh │ ├── follow │ │ ├── test.sh │ │ └── test.yaml │ ├── no-reservation │ │ ├── test.yaml │ │ └── test.sh │ ├── webhook_ready.sh │ ├── cert-manager.sh │ ├── install_dependencies.sh │ ├── virtual-node-labels │ │ └── test.sh │ ├── no-rogue-finalizer │ │ └── test.sh │ ├── ingress │ │ ├── test.sh │ │ └── test.yaml │ ├── kind.sh │ ├── argo.sh │ ├── e2e.sh │ └── admiralty.sh └── test.sh ├── charts └── multicluster-scheduler │ ├── templates │ ├── sa.yaml │ ├── issuer.yaml │ ├── quota.yaml │ ├── post-delete │ │ ├── sa.yaml │ │ ├── crb.yaml │ │ ├── cr.yaml │ │ └── job.yaml │ ├── svc.yaml │ ├── cert.yaml │ ├── crds │ │ ├── source.yaml │ │ ├── clustersummary.yaml │ │ ├── clustersource.yaml │ │ ├── target.yaml │ │ ├── podchaperon.yaml │ │ └── clustertarget.yaml │ ├── cm.yaml │ ├── webhook.yaml │ └── _helpers.tpl │ ├── Chart.yaml │ └── README.md ├── .github ├── dependabot.yml └── workflows │ └── go.yml ├── docs ├── user_guide │ ├── integrations │ │ ├── argo_cd.md │ │ └── argo_workflows.md │ └── pod_configuration.md ├── operator_guide │ ├── safely_drain_cluster.md │ └── installation.md ├── introduction.md └── concepts │ ├── topologies.md │ └── authentication.md ├── examples ├── argo-workflows │ ├── blog-scenario-a-singlecluster.yaml │ ├── blog-scenario-a-multicluster.yaml │ ├── blog-scenario-b.yaml │ └── _service-account.yaml └── multicluster-deployment.yaml ├── hack ├── boilerplate.go.txt ├── tools.go ├── verify-codegen.sh ├── update-codegen.sh └── version-bump.sh ├── .gitignore ├── pkg ├── apis │ ├── multicluster │ │ ├── group.go │ │ └── v1alpha1 │ │ │ ├── doc.go │ │ │ ├── zz_generated.conversion.go │ │ │ ├── podchaperon_types.go │ │ │ ├── clustersummary_types.go │ │ │ ├── source_types.go │ │ │ ├── register.go │ │ │ ├── target_types.go │ │ │ ├── clustersource_types.go │ │ │ └── clustertarget_types.go │ ├── addtoscheme_multicluster_v1alpha1.go │ └── apis.go ├── vk │ ├── http │ │ ├── opts.go │ │ └── http.go │ └── node │ │ ├── provider.go │ │ ├── run.go │ │ └── node.go ├── generated │ ├── clientset │ │ └── versioned │ │ │ ├── fake │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ └── clientset_generated.go │ │ │ ├── scheme │ │ │ ├── doc.go │ │ │ └── register.go │ │ │ └── typed │ │ │ └── multicluster │ │ │ └── v1alpha1 │ │ │ ├── fake │ │ │ ├── doc.go │ │ │ └── fake_multicluster_client.go │ │ │ ├── doc.go │ │ │ └── generated_expansion.go │ ├── informers │ │ └── externalversions │ │ │ ├── internalinterfaces │ │ │ └── factory_interfaces.go │ │ │ ├── multicluster │ │ │ ├── interface.go │ │ │ └── v1alpha1 │ │ │ │ ├── interface.go │ │ │ │ ├── source.go │ │ │ │ ├── target.go │ │ │ │ └── clustersource.go │ │ │ └── generic.go │ └── listers │ │ └── multicluster │ │ └── v1alpha1 │ │ ├── expansion_generated.go │ │ ├── clustersource.go │ │ ├── clustertarget.go │ │ ├── clustersummary.go │ │ ├── source.go │ │ └── target.go ├── common │ ├── util.go │ └── constants.go ├── controller │ ├── errors.go │ └── handlers.go ├── scheduler_plugins │ └── proxy │ │ ├── utils.go │ │ └── utils_test.go ├── model │ ├── virtualnode │ │ └── model.go │ ├── proxypod │ │ └── model.go │ └── delegatepod │ │ └── model_test.go ├── leaderelection │ └── leaderelection.go ├── name │ ├── name.go │ └── name_test.go └── controllers │ ├── resources │ └── upstream_test.go │ └── feedback │ └── controller_test.go ├── release ├── images.sh ├── chart.sh └── image.sh ├── pipeline.sh ├── README.md ├── cmd ├── scheduler │ └── main.go └── restarter │ └── main.go └── CONTRIBUTING.md /test/e2e/priorityclass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduling.k8s.io/v1 2 | kind: PriorityClass 3 | metadata: 4 | name: default 5 | value: 1000 6 | globalDefault: true 7 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/sa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "fullname" . }} 5 | labels: {{ include "labels" . | nindent 4 }} 6 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | ./hack/verify-codegen.sh 5 | go vet ./pkg/... ./cmd/... 6 | go test -v ./pkg/... ./cmd/... # -coverprofile cover.out 7 | # TODO save cover.out somewhere 8 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/issuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Issuer 3 | metadata: 4 | name: {{ include "fullname" . }} 5 | labels: {{ include "labels" . | nindent 4 }} 6 | spec: 7 | selfSigned: {} 8 | -------------------------------------------------------------------------------- /test/e2e/aliases.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | k() { KUBECONFIG=kubeconfig-cluster$1 kubectl "${@:2}"; } 4 | h() { KUBECONFIG=kubeconfig-cluster$1 helm "${@:2}"; } 5 | wk() { KUBECONFIG=kubeconfig-cluster$1 watch kubectl "${@:2}"; } 6 | -------------------------------------------------------------------------------- /test/e2e/delete-chaperon/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-delete-chaperon 5 | annotations: 6 | multicluster.admiralty.io/elect: "" 7 | spec: 8 | containers: 9 | - name: pause 10 | image: gcr.io/google_containers/pause 11 | -------------------------------------------------------------------------------- /test/e2e/values.yaml: -------------------------------------------------------------------------------- 1 | controllerManager: 2 | securityContext: 3 | runAsUser: 1000 4 | 5 | scheduler: 6 | securityContext: 7 | runAsUser: 1000 8 | 9 | postDeleteJob: 10 | securityContext: 11 | runAsUser: 1000 12 | 13 | restarter: 14 | securityContext: 15 | runAsUser: 1000 16 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/quota.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ResourceQuota 3 | metadata: 4 | name: {{ include "fullname" . }} 5 | spec: 6 | scopeSelector: 7 | matchExpressions: 8 | - operator: In 9 | scopeName: PriorityClass 10 | values: 11 | - system-cluster-critical 12 | -------------------------------------------------------------------------------- /test/e2e/delete-delegate/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-delete-delegate 5 | labels: 6 | app: delete-delegate 7 | annotations: 8 | multicluster.admiralty.io/elect: "" 9 | spec: 10 | nodeSelector: 11 | a: b 12 | containers: 13 | - name: pause 14 | image: gcr.io/google_containers/pause 15 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/post-delete/sa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "fullname" . }}-post-delete-hook 5 | labels: {{ include "labels" . | nindent 4 }} 6 | annotations: 7 | "helm.sh/hook": post-delete 8 | "helm.sh/hook-weight": "0" 9 | "helm.sh/hook-delete-policy": hook-succeeded 10 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Dependencies listed in go.mod 4 | - package-ecosystem: "gomod" 5 | directory: "/" # Location of package manifests 6 | schedule: 7 | interval: "weekly" 8 | 9 | # Dependencies listed in .github/workflows/*.yml 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | -------------------------------------------------------------------------------- /test/e2e/exec/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: exec 5 | spec: 6 | template: 7 | metadata: 8 | annotations: 9 | multicluster.admiralty.io/elect: "" 10 | spec: 11 | nodeSelector: 12 | a: b 13 | restartPolicy: Never 14 | containers: 15 | - name: logs 16 | image: busybox 17 | command: [ "/bin/sh", "-c", "sleep 100" ] -------------------------------------------------------------------------------- /test/e2e/logs/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: logs 5 | spec: 6 | template: 7 | metadata: 8 | annotations: 9 | multicluster.admiralty.io/elect: "" 10 | spec: 11 | nodeSelector: 12 | a: b 13 | restartPolicy: Never 14 | containers: 15 | - name: logs 16 | image: busybox 17 | command: [ "/bin/sh", "-c", "echo bonjour" ] -------------------------------------------------------------------------------- /test/e2e/cleanup/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: cleanup 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: cleanup 9 | template: 10 | metadata: 11 | labels: 12 | app: cleanup 13 | annotations: 14 | multicluster.admiralty.io/elect: "" 15 | spec: 16 | containers: 17 | - name: pause 18 | image: gcr.io/google_containers/pause 19 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.debug.controllerManager }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "fullname" . }} 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | selector: {{ include "selectorLabels" . | nindent 4 }} 9 | component: controller-manager 10 | ports: 11 | - port: 443 12 | protocol: TCP 13 | targetPort: 9443 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /test/e2e/follow/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | source test/e2e/aliases.sh 5 | 6 | follow_test() { 7 | i=$1 8 | j=$2 9 | 10 | k $j label node --all a=b --overwrite 11 | k $i apply -f test/e2e/follow/test.yaml 12 | k $i wait job/follow --for=condition=Complete 13 | k $i delete -f test/e2e/follow/test.yaml 14 | k $j label node --all a- 15 | } 16 | 17 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 18 | follow_test "${@}" 19 | fi 20 | -------------------------------------------------------------------------------- /docs/user_guide/integrations/argo_cd.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Argo CD 3 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/user_guide/integrations/argo_cd.md 4 | --- 5 | 6 | :::caution Under Construction 7 | While we work on this document, check out [ITNEXT's blog post](https://itnext.io/multicluster-scheduler-argo-workflows-across-kubernetes-clusters-ea98016499ca) describing an integration with [Argo CD](https://argoproj.github.io/projects/argo-cd) (scroll down to the relevant section). 8 | ::: 9 | -------------------------------------------------------------------------------- /test/e2e/no-reservation/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: no-reservation 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: no-reservation 9 | template: 10 | metadata: 11 | labels: 12 | app: no-reservation 13 | annotations: 14 | multicluster.admiralty.io/elect: "" 15 | multicluster.admiralty.io/no-reservation: "" 16 | spec: 17 | containers: 18 | - name: pause 19 | image: gcr.io/google_containers/pause 20 | -------------------------------------------------------------------------------- /docs/user_guide/integrations/argo_workflows.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Argo Workflows 3 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/user_guide/integrations/argo_workflows.md 4 | --- 5 | 6 | :::caution Under Construction 7 | While we work on this document, check out [Admiralty's blog post](https://admiralty.io/blog/running-argo-workflows-across-multiple-kubernetes-clusters/) demonstrating how to run an Argo workflow across clusters to combine data from different regions or clouds, or better utilize resources. 8 | ::: 9 | -------------------------------------------------------------------------------- /examples/argo-workflows/blog-scenario-a-singlecluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: singlecluster-parallel- 5 | spec: 6 | entrypoint: singlecluster-parallel 7 | templates: 8 | - name: singlecluster-parallel 9 | steps: 10 | - - name: sleep 11 | template: sleep 12 | withItems: [0, 1, 2, 3, 4, 5, 6, 7, 9, 10] 13 | - name: sleep 14 | container: 15 | image: busybox 16 | command: [ sleep, "10" ] 17 | resources: 18 | requests: 19 | cpu: 100m # Note: Argo sidecar adds another 100m 20 | -------------------------------------------------------------------------------- /examples/multicluster-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx 5 | spec: 6 | replicas: 10 7 | selector: 8 | matchLabels: 9 | app: nginx 10 | template: 11 | metadata: 12 | labels: 13 | app: nginx 14 | annotations: 15 | multicluster.admiralty.io/elect: "" 16 | spec: 17 | # nodeSelector: 18 | # foo: bar 19 | containers: 20 | - name: nginx 21 | image: nginx 22 | resources: 23 | requests: 24 | cpu: 100m 25 | memory: 32Mi 26 | ports: 27 | - containerPort: 80 28 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/post-delete/crb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ include "fullname" . }}-post-delete-hook 5 | labels: {{ include "labels" . | nindent 4 }} 6 | annotations: 7 | "helm.sh/hook": post-delete 8 | "helm.sh/hook-weight": "0" 9 | "helm.sh/hook-delete-policy": hook-succeeded 10 | roleRef: 11 | apiGroup: rbac.authorization.k8s.io 12 | kind: ClusterRole 13 | name: {{ include "fullname" . }}-post-delete-hook 14 | subjects: 15 | - kind: ServiceAccount 16 | name: {{ include "fullname" . }}-post-delete-hook 17 | namespace: {{ .Release.Namespace }} 18 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/cert.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Certificate 3 | metadata: 4 | name: {{ include "fullname" . }} 5 | labels: {{ include "labels" . | nindent 4 }} 6 | spec: 7 | commonName: {{ include "fullname" . }}.{{ .Release.Namespace }}.svc 8 | dnsNames: 9 | - {{ include "fullname" . }}.{{ .Release.Namespace }}.svc 10 | - {{ include "fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local 11 | {{- if .Values.debug.controllerManager }} 12 | ipAddresses: 13 | - 172.17.0.1 14 | {{- end }} 15 | secretName: {{ include "fullname" . }}-cert 16 | issuerRef: 17 | name: {{ include "fullname" . }} 18 | -------------------------------------------------------------------------------- /examples/argo-workflows/blog-scenario-a-multicluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: multicluster-parallel- 5 | spec: 6 | entrypoint: multicluster-parallel 7 | templates: 8 | - name: multicluster-parallel 9 | steps: 10 | - - name: sleep 11 | template: sleep 12 | withItems: [0, 1, 2, 3, 4, 5, 6, 7, 9, 10] 13 | - name: sleep 14 | container: 15 | image: busybox 16 | command: [ sleep, "10" ] 17 | resources: 18 | requests: 19 | cpu: 100m # Note: Argo sidecar adds another 100m 20 | metadata: 21 | annotations: 22 | multicluster.admiralty.io/elect: "" 23 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Kubernetes Generated files - skip generated files, except for vendored files 17 | 18 | !vendor/**/zz_generated.* 19 | 20 | # editor and IDE paraphernalia 21 | .idea 22 | *.swp 23 | *.swo 24 | *~ 25 | 26 | # Mac OS X 27 | .DS_Store 28 | 29 | .vscode/ 30 | 31 | # build artifacts 32 | _out 33 | 34 | # test binaries 35 | argo 36 | kubemcsa 37 | 38 | # test kubeconfigs 39 | kubeconfig* 40 | 41 | scratch/ 42 | cluster-dump/ 43 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/post-delete/cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: {{ include "fullname" . }}-post-delete-hook 5 | labels: {{ include "labels" . | nindent 4 }} 6 | annotations: 7 | "helm.sh/hook": post-delete 8 | "helm.sh/hook-weight": "0" 9 | "helm.sh/hook-delete-policy": hook-succeeded 10 | rules: 11 | - apiGroups: 12 | - "" 13 | resources: 14 | - pods 15 | - services 16 | - configmaps 17 | - secrets 18 | verbs: 19 | - list 20 | - patch 21 | - apiGroups: 22 | - extensions 23 | - networking.k8s.io 24 | resources: 25 | - ingresses 26 | verbs: 27 | - list 28 | - patch 29 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/group.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Multicluster-Scheduler Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package multicluster contains multicluster API versions 18 | package multicluster 19 | -------------------------------------------------------------------------------- /pkg/vk/http/opts.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package http 18 | 19 | const ( 20 | DefaultKubeletAddr = ":10250" 21 | ) 22 | -------------------------------------------------------------------------------- /hack/tools.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // +build tools 18 | 19 | package hack 20 | 21 | import ( 22 | _ "k8s.io/code-generator" 23 | ) 24 | -------------------------------------------------------------------------------- /test/e2e/follow/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: follow 5 | data: 6 | MY_CONFIG: foo 7 | --- 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: follow 12 | stringData: 13 | MY_SECRET: bar 14 | --- 15 | apiVersion: batch/v1 16 | kind: Job 17 | metadata: 18 | name: follow 19 | spec: 20 | template: 21 | metadata: 22 | annotations: 23 | multicluster.admiralty.io/elect: "" 24 | spec: 25 | nodeSelector: 26 | a: b 27 | restartPolicy: Never 28 | containers: 29 | - name: follow 30 | image: busybox 31 | command: ["/bin/sh", "-c", "[ $MY_CONFIG = foo ] && [ $MY_SECRET = bar ]"] 32 | envFrom: 33 | - configMapRef: 34 | name: follow 35 | - secretRef: 36 | name: follow -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated fake clientset. 20 | package fake 21 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: admiralty 3 | version: 0.17.0 4 | #kubeVersion: A SemVer range of compatible Kubernetes versions (optional) 5 | description: A system of Kubernetes controllers that intelligently schedules workloads across clusters. 6 | type: application 7 | #keywords: 8 | # - A list of keywords about this project (optional) 9 | home: https://github.com/admiraltyio/admiralty 10 | #sources: 11 | # - https://github.com/admiraltyio/admiralty 12 | #maintainers: # (optional) 13 | # - name: The maintainer's name (required for each maintainer) 14 | # email: The maintainer's email (optional for each maintainer) 15 | # url: A URL for the maintainer (optional for each maintainer) 16 | icon: https://admiralty.io/icons/icon-144x144.png 17 | appVersion: 0.17.0 18 | #deprecated: Whether this chart is deprecated (optional, boolean) 19 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package contains the scheme of the automatically generated clientset. 20 | package scheme 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/multicluster/v1alpha1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // Package fake has the automatically generated clients. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/multicluster/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated typed clients. 20 | package v1alpha1 21 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/crds/source.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: sources.multicluster.admiralty.io 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | group: multicluster.admiralty.io 9 | names: 10 | kind: Source 11 | plural: sources 12 | shortNames: 13 | - src 14 | scope: Namespaced 15 | versions: 16 | - name: v1alpha1 17 | served: true 18 | storage: true 19 | subresources: 20 | status: { } 21 | schema: 22 | openAPIV3Schema: 23 | type: object 24 | properties: 25 | spec: 26 | type: object 27 | properties: 28 | userName: 29 | type: string 30 | serviceAccountName: 31 | type: string 32 | status: 33 | type: object 34 | -------------------------------------------------------------------------------- /release/images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | imgs=( 21 | admiralty-agent 22 | admiralty-remove-finalizers 23 | admiralty-scheduler 24 | admiralty-restarter 25 | ) 26 | 27 | for img in "${imgs[@]}"; do 28 | IMG="$img" ./release/image.sh 29 | done 30 | -------------------------------------------------------------------------------- /docs/user_guide/pod_configuration.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Pod Configuration 3 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/user_guide/pod_configuration.md 4 | --- 5 | 6 | 7 | ## Label Prefixing 8 | Admiralty prefixes delegate pod labels with `multicluster.admiralty.io/` in the target clusters. This behavior is useful in bursting cluster topologies, where the source cluster is also a target cluster, so as not to confuse controllers of proxy pods, e.g., replicasets. The behavior 9 | can be overridden per pod through the `multicluster.admiralty.io/no-prefix-label-regexp` annotation. This is useful to 10 | support components that have functionality that relies on pod labels. For example, webhooks or monitoring resources. 11 | 12 | 13 | :::tip 14 | One use-case is to prevent prefixing the Kueue queue name label. This can be achieved through: 15 | ``` 16 | multicluster.admiralty.io/no-prefix-label-regexp="^kueue\.x-k8s\.io\/queue-name" 17 | ``` 18 | ::: 19 | 20 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/crds/clustersummary.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: clustersummaries.multicluster.admiralty.io 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | group: multicluster.admiralty.io 9 | names: 10 | kind: ClusterSummary 11 | plural: clustersummaries 12 | shortNames: 13 | - mcsum 14 | scope: Cluster 15 | versions: 16 | - name: v1alpha1 17 | served: true 18 | storage: true 19 | subresources: 20 | status: { } 21 | schema: 22 | openAPIV3Schema: 23 | type: object 24 | properties: 25 | capacity: 26 | type: object 27 | additionalProperties: 28 | x-kubernetes-int-or-string: true 29 | allocatable: 30 | type: object 31 | additionalProperties: 32 | x-kubernetes-int-or-string: true 33 | -------------------------------------------------------------------------------- /pkg/apis/addtoscheme_multicluster_v1alpha1.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Multicluster-Scheduler Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apis 18 | 19 | import ( 20 | "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 21 | ) 22 | 23 | func init() { 24 | // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back 25 | AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme) 26 | } 27 | -------------------------------------------------------------------------------- /examples/argo-workflows/blog-scenario-b.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: multicluster-dag- 5 | spec: 6 | entrypoint: multicluster-dag 7 | templates: 8 | - name: multicluster-dag 9 | dag: 10 | tasks: 11 | - name: A 12 | template: sleep 13 | - name: B 14 | template: sleep-remote 15 | arguments: 16 | parameters: 17 | - name: clustername 18 | value: cluster2 19 | - name: C 20 | dependencies: [A, B] 21 | template: sleep 22 | - name: sleep 23 | container: 24 | image: busybox 25 | command: [ sleep, "10" ] 26 | - name: sleep-remote 27 | inputs: 28 | parameters: 29 | - name: clustername 30 | container: 31 | image: busybox 32 | command: [ sleep, "10" ] 33 | metadata: 34 | annotations: 35 | multicluster.admiralty.io/elect: "" 36 | multicluster.admiralty.io/clustername: "{{inputs.parameters.clustername}}" -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Multicluster-Scheduler Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the multicluster v1alpha1 API group 18 | // +k8s:openapi-gen=true 19 | // +k8s:deepcopy-gen=package,register 20 | // +k8s:conversion-gen=admiralty.io/multicluster-scheduler/pkg/apis/multicluster 21 | // +k8s:defaulter-gen=TypeMeta 22 | // +groupName=multicluster.admiralty.io 23 | package v1alpha1 24 | -------------------------------------------------------------------------------- /pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | echo "test" 21 | test/test.sh 22 | echo "build" 23 | build/build.sh 24 | echo "e2e test" 25 | test/e2e/e2e.sh 26 | 27 | if [ "${VERSION:-dev}" = dev ]; then 28 | exit 0 29 | fi 30 | 31 | echo "release images" 32 | release/images.sh 33 | echo "release chart" 34 | release/chart.sh 35 | # TODO: create release on GitHub 36 | -------------------------------------------------------------------------------- /test/e2e/webhook_ready.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | source test/e2e/aliases.sh 5 | 6 | webhook_ready() { 7 | cluster_id=$1 8 | namespace=$2 9 | deployment_name=$3 10 | config_name=$4 11 | secret_name=$5 12 | 13 | echo "waiting for webhook deployment to be available..." 14 | k $cluster_id wait --for condition=available --timeout=120s deployment $deployment_name -n $namespace 15 | 16 | echo -n "waiting for webhook configuration CA bundle to match secret..." 17 | while :; do 18 | secret_cert=$(k $cluster_id get secret $secret_name -n $namespace -o json | jq -r '.data["ca.crt"]') 19 | webhook_cert=$(k $cluster_id get mutatingwebhookconfiguration $config_name -n $namespace -o json | jq -r .webhooks[0].clientConfig.caBundle) 20 | if [ "$secret_cert" == "$webhook_cert" ]; then 21 | echo 22 | break 23 | fi 24 | sleep 1 25 | echo -n "." 26 | done 27 | 28 | # still something is missing 29 | # maybe https://github.com/kubernetes-sigs/controller-runtime/issues/723 30 | # so for now... 31 | sleep 10 32 | } 33 | -------------------------------------------------------------------------------- /pkg/common/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import ( 20 | "strings" 21 | ) 22 | 23 | func SplitLabelsOrAnnotations(m map[string]string) (map[string]string, map[string]string) { 24 | mc := make(map[string]string) 25 | other := make(map[string]string) 26 | for k, v := range m { 27 | if strings.HasPrefix(k, KeyPrefix) { 28 | mc[k] = v 29 | } else { 30 | other[k] = v 31 | } 32 | } 33 | return mc, other 34 | } 35 | -------------------------------------------------------------------------------- /pkg/apis/apis.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package apis contains Kubernetes API groups. 18 | package apis 19 | 20 | import ( 21 | "k8s.io/apimachinery/pkg/runtime" 22 | ) 23 | 24 | // AddToSchemes may be used to add all resources defined in the project to a Scheme 25 | var AddToSchemes runtime.SchemeBuilder 26 | 27 | // AddToScheme adds all Resources to the Scheme 28 | func AddToScheme(s *runtime.Scheme) error { 29 | return AddToSchemes.AddToScheme(s) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/multicluster/v1alpha1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | type ClusterSourceExpansion interface{} 22 | 23 | type ClusterSummaryExpansion interface{} 24 | 25 | type ClusterTargetExpansion interface{} 26 | 27 | type PodChaperonExpansion interface{} 28 | 29 | type SourceExpansion interface{} 30 | 31 | type TargetExpansion interface{} 32 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/crds/clustersource.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: clustersources.multicluster.admiralty.io 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | group: multicluster.admiralty.io 9 | names: 10 | kind: ClusterSource 11 | plural: clustersources 12 | shortNames: 13 | - csrc 14 | scope: Cluster 15 | versions: 16 | - name: v1alpha1 17 | served: true 18 | storage: true 19 | subresources: 20 | status: { } 21 | schema: 22 | openAPIV3Schema: 23 | type: object 24 | properties: 25 | spec: 26 | type: object 27 | properties: 28 | userName: 29 | type: string 30 | serviceAccount: 31 | type: object 32 | properties: 33 | name: 34 | type: string 35 | namespace: 36 | type: string 37 | status: 38 | type: object 39 | -------------------------------------------------------------------------------- /pkg/controller/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controller 18 | 19 | import "strings" 20 | 21 | // from k8s.io/apiserver/pkg/registry/generic/registry/store.go#L201 (no need to import package fro just the one constant) 22 | var optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" 23 | 24 | func IsOptimisticLockError(err error) bool { 25 | return strings.Contains(err.Error(), optimisticLockErrorMsg) 26 | } 27 | -------------------------------------------------------------------------------- /test/e2e/logs/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2020 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | logs_test() { 23 | i=$1 24 | j=$2 25 | 26 | k $j label node --all a=b --overwrite 27 | k $i apply -f test/e2e/logs/test.yaml 28 | k $i wait job/logs --for=condition=Complete 29 | k $i logs job/logs | grep bonjour 30 | k $i delete -f test/e2e/logs/test.yaml 31 | k $j label node --all a- 32 | } 33 | 34 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 35 | logs_test "${@}" 36 | fi 37 | -------------------------------------------------------------------------------- /pkg/scheduler_plugins/proxy/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package proxy 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | 22 | "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 23 | ) 24 | 25 | func isCandidatePodUnschedulable(c *v1alpha1.PodChaperon) bool { 26 | for _, cond := range c.Status.Conditions { 27 | if cond.Type == v1.PodScheduled && cond.Status == v1.ConditionFalse && (cond.Reason == v1.PodReasonUnschedulable || cond.Reason == v1.PodReasonSchedulingGated) { 28 | return true 29 | } 30 | } 31 | return false 32 | } 33 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "fullname" . }} 5 | labels: {{ include "labels" . | nindent 4 }} 6 | data: 7 | proxy-scheduler-config: | 8 | apiVersion: kubescheduler.config.k8s.io/v1 9 | kind: KubeSchedulerConfiguration 10 | leaderElection: 11 | leaderElect: true 12 | resourceName: admiralty-proxy-scheduler 13 | resourceNamespace: {{ .Release.Namespace }} 14 | resourceLock: leases 15 | profiles: 16 | - schedulerName: admiralty-proxy 17 | plugins: 18 | multiPoint: 19 | enabled: 20 | - name: proxy 21 | filter: 22 | enabled: 23 | - name: proxy 24 | candidate-scheduler-config: | 25 | apiVersion: kubescheduler.config.k8s.io/v1 26 | kind: KubeSchedulerConfiguration 27 | leaderElection: 28 | leaderElect: true 29 | resourceName: admiralty-candidate-scheduler 30 | resourceNamespace: {{ .Release.Namespace }} 31 | resourceLock: leases 32 | profiles: 33 | - schedulerName: admiralty-candidate 34 | plugins: 35 | multiPoint: 36 | enabled: 37 | - name: candidate 38 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/crds/target.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: targets.multicluster.admiralty.io 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | group: multicluster.admiralty.io 9 | names: 10 | kind: Target 11 | plural: targets 12 | shortNames: 13 | - tg 14 | scope: Namespaced 15 | versions: 16 | - name: v1alpha1 17 | served: true 18 | storage: true 19 | subresources: 20 | status: { } 21 | schema: 22 | openAPIV3Schema: 23 | type: object 24 | properties: 25 | spec: 26 | type: object 27 | properties: 28 | self: 29 | type: boolean 30 | kubeconfigSecret: 31 | type: object 32 | properties: 33 | name: 34 | type: string 35 | key: 36 | type: string 37 | context: 38 | type: string 39 | excludedLabelsRegexp: 40 | type: string 41 | status: 42 | type: object 43 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/zz_generated.conversion.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | * Copyright The Multicluster-Scheduler Authors. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | 20 | // Code generated by conversion-gen. DO NOT EDIT. 21 | 22 | package v1alpha1 23 | 24 | import ( 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | ) 27 | 28 | func init() { 29 | localSchemeBuilder.Register(RegisterConversions) 30 | } 31 | 32 | // RegisterConversions adds conversion functions to the given scheme. 33 | // Public to allow building arbitrary schemes. 34 | func RegisterConversions(s *runtime.Scheme) error { 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Admiralty 2 | 3 | _formerly multicluster-scheduler_ 4 | 5 | Admiralty is a system of Kubernetes controllers that intelligently schedules workloads across clusters. It is simple to use and simple to integrate with other tools. 6 | 7 | The documentation hosted at https://admiralty.io/docs/ is sourced from this repository. The links below point to the local Markdown files. Use them if you're browsing this repo without Internet access; otherwise, **the [hosted version](https://admiralty.io/docs/) is easier to navigate**. 8 | 9 | - [Introduction](docs/introduction.md) 10 | - [Quick Start](docs/quick_start.md) 11 | - Concepts 12 | - [Multi-Cluster Topologies](docs/concepts/topologies.md) 13 | - [Cross-Cluster Authentication](docs/concepts/authentication.md) 14 | - [Multi-Cluster Scheduling](docs/concepts/scheduling.md) 15 | - Operator Guide 16 | - [Installation](docs/operator_guide/installation.md) 17 | - [Configuring Authentication](docs/operator_guide/authentication.md) 18 | - [Configuring Scheduling](docs/operator_guide/scheduling.md) 19 | - [Contributor Guide](CONTRIBUTING.md) 20 | - [Release Notes](CHANGELOG.md) 21 | - API Reference 22 | - [Helm Chart](charts/multicluster-scheduler/README.md) 23 | - [License](LICENSE) 24 | -------------------------------------------------------------------------------- /test/e2e/exec/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2020 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | exec_test() { 23 | i=$1 24 | j=$2 25 | 26 | k $j label node --all a=b --overwrite 27 | k $i apply -f test/e2e/exec/test.yaml 28 | while [ $(k $i get pod -l job-name=exec | wc -l) = 0 ]; do sleep 1; done 29 | k $i wait pod -l job-name=exec --for=condition=ContainersReady 30 | k $i exec job/exec ls | grep bin 31 | k $i delete -f test/e2e/exec/test.yaml 32 | k $j label node --all a- 33 | } 34 | 35 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 36 | exec_test "${@}" 37 | fi 38 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/webhook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: {{ include "fullname" . }} 5 | labels: {{ include "labels" . | nindent 4 }} 6 | annotations: 7 | cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "fullname" . }} 8 | webhooks: 9 | - clientConfig: 10 | caBundle: Cg== 11 | {{- if .Values.debug.controllerManager }} 12 | url: "https://172.17.0.1:9443/mutate--v1-pod" 13 | {{- else }} 14 | service: 15 | name: {{ include "fullname" . }} 16 | namespace: {{ .Release.Namespace }} 17 | path: /mutate--v1-pod 18 | {{- end }} 19 | failurePolicy: Fail 20 | name: {{ include "fullname" . }}.multicluster.admiralty.io 21 | namespaceSelector: 22 | matchLabels: 23 | multicluster-scheduler: enabled 24 | rules: 25 | - apiGroups: 26 | - "" 27 | apiVersions: 28 | - v1 29 | operations: 30 | - CREATE 31 | resources: 32 | - pods 33 | scope: '*' 34 | sideEffects: None 35 | admissionReviewVersions: [v1beta1] 36 | reinvocationPolicy: {{ .Values.webhook.reinvocationPolicy }} 37 | -------------------------------------------------------------------------------- /test/e2e/cert-manager.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | cert_manager_setup_once() { 23 | helm repo add jetstack https://charts.jetstack.io 24 | helm repo update 25 | } 26 | 27 | cert_manager_setup() { 28 | i=$1 29 | 30 | h $i upgrade --install cert-manager jetstack/cert-manager \ 31 | --namespace cert-manager --create-namespace \ 32 | --version v1.13.1 --set installCRDs=true \ 33 | --wait --debug --timeout=1m 34 | } 35 | 36 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 37 | cert_manager_setup_one 38 | cert_manager_setup $1 39 | fi 40 | -------------------------------------------------------------------------------- /release/chart.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | # constants 21 | default_registry="public.ecr.aws/admiralty" 22 | 23 | # environment variables 24 | # required 25 | VERSION="${VERSION}" 26 | 27 | # delete any leftover packaged charts 28 | # (otherwise dates in index for their versions would be modified) 29 | rm -f _out/admiralty-*.tgz 30 | 31 | helm package charts/multicluster-scheduler -d _out 32 | 33 | aws ecr-public get-login-password --region us-east-1 | helm registry login --username AWS --password-stdin public.ecr.aws 34 | 35 | helm push _out/admiralty-"${VERSION}".tgz oci://$default_registry 36 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/crds/podchaperon.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: podchaperons.multicluster.admiralty.io 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | group: multicluster.admiralty.io 9 | names: 10 | kind: PodChaperon 11 | plural: podchaperons 12 | shortNames: 13 | - chap 14 | scope: Namespaced 15 | versions: 16 | - name: v1alpha1 17 | served: true 18 | storage: true 19 | subresources: 20 | status: { } 21 | schema: 22 | openAPIV3Schema: 23 | type: object 24 | properties: 25 | spec: 26 | type: object 27 | x-kubernetes-preserve-unknown-fields: true 28 | # TODO generate 29 | status: 30 | type: object 31 | x-kubernetes-preserve-unknown-fields: true 32 | # TODO generate 33 | additionalPrinterColumns: 34 | - name: reserved 35 | type: string 36 | jsonPath: .metadata.annotations.multicluster\.admiralty\.io/is-reserved 37 | - name: allowed 38 | type: string 39 | jsonPath: .metadata.annotations.multicluster\.admiralty\.io/is-allowed 40 | -------------------------------------------------------------------------------- /examples/argo-workflows/_service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: argo-workflow 5 | rules: 6 | # pod get/watch is used to identify the container IDs of the current pod 7 | # pod patch is used to annotate the step's outputs back to controller (e.g. artifact location) 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - pods 12 | verbs: 13 | - get 14 | - watch 15 | - patch 16 | # logs get/watch are used to get the pods logs for script outputs, and for log archival 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - pods/log 21 | verbs: 22 | - get 23 | - watch 24 | # secrets get is used to retrieve credentials to artifact repository. NOTE: starting n Argo v2.3, 25 | # the API secret access will be removed in favor of volume mounting the secrets to the workflow pod 26 | # (issue #1072) 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - secrets 31 | verbs: 32 | - get 33 | --- 34 | apiVersion: v1 35 | kind: ServiceAccount 36 | metadata: 37 | name: argo-workflow 38 | --- 39 | apiVersion: rbac.authorization.k8s.io/v1 40 | kind: RoleBinding 41 | metadata: 42 | name: argo-workflow 43 | roleRef: 44 | apiGroup: rbac.authorization.k8s.io 45 | kind: Role 46 | name: argo-workflow 47 | subjects: 48 | - kind: ServiceAccount 49 | name: argo-workflow 50 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/crds/clustertarget.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: clustertargets.multicluster.admiralty.io 6 | labels: {{ include "labels" . | nindent 4 }} 7 | spec: 8 | group: multicluster.admiralty.io 9 | names: 10 | kind: ClusterTarget 11 | plural: clustertargets 12 | shortNames: 13 | - ctg 14 | scope: Cluster 15 | versions: 16 | - name: v1alpha1 17 | served: true 18 | storage: true 19 | subresources: 20 | status: { } 21 | schema: 22 | openAPIV3Schema: 23 | type: object 24 | properties: 25 | spec: 26 | type: object 27 | properties: 28 | self: 29 | type: boolean 30 | kubeconfigSecret: 31 | type: object 32 | properties: 33 | name: 34 | type: string 35 | namespace: 36 | type: string 37 | key: 38 | type: string 39 | context: 40 | type: string 41 | excludedLabelsRegexp: 42 | type: string 43 | status: 44 | type: object 45 | -------------------------------------------------------------------------------- /test/e2e/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2023 The Multicluster-Scheduler Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | set -euo pipefail 18 | 19 | # from https://kubernetes.io/docs/tasks/tools/install-kubectl/ 20 | 21 | curl -LO https://dl.k8s.io/release/v1.30.4/bin/linux/amd64/kubectl 22 | chmod +x ./kubectl 23 | sudo mv ./kubectl /usr/local/bin/kubectl 24 | 25 | # from https://kind.sigs.k8s.io/docs/user/quick-start/ 26 | 27 | curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 28 | chmod +x ./kind 29 | sudo mv ./kind /usr/local/bin/kind 30 | 31 | # from https://helm.sh/docs/intro/install/ 32 | 33 | curl -LO https://get.helm.sh/helm-v3.11.1-linux-amd64.tar.gz 34 | tar -zxvf helm-v3.11.1-linux-amd64.tar.gz 35 | sudo mv linux-amd64/helm /usr/local/bin/helm 36 | -------------------------------------------------------------------------------- /test/e2e/delete-chaperon/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | delete-chaperon_test() { 23 | i=$1 24 | 25 | k $i apply -f test/e2e/delete-chaperon/test.yaml 26 | k $i wait pod test-delete-chaperon --for=condition=PodScheduled 27 | nodeName="$(k $i get pod test-delete-chaperon -o json | jq -er '.spec.nodeName')" 28 | j="${nodeName: -1}" 29 | uid="$(k $i get pod test-delete-chaperon -o json | jq -er '.metadata.uid')" 30 | k $j delete podchaperon -l multicluster.admiralty.io/parent-uid="$uid" --wait --timeout=30s 31 | k $i wait pod test-delete-chaperon --for=delete 32 | } 33 | 34 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 35 | delete-chaperon_test "${@}" 36 | fi 37 | -------------------------------------------------------------------------------- /cmd/scheduler/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "os" 21 | 22 | "admiralty.io/multicluster-scheduler/pkg/scheduler_plugins/candidate" 23 | "admiralty.io/multicluster-scheduler/pkg/scheduler_plugins/proxy" 24 | "k8s.io/component-base/cli" 25 | scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" 26 | ) 27 | 28 | func main() { 29 | // BEWARE candidate and proxy must run in different processes, because a scheduler only processes one pod at a time 30 | // and proxy waits on candidates in filter plugin 31 | 32 | command := scheduler.NewSchedulerCommand( 33 | scheduler.WithPlugin(candidate.Name, candidate.New), 34 | scheduler.WithPlugin(proxy.Name, proxy.New)) 35 | 36 | code := cli.Run(command) 37 | os.Exit(code) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/controller/handlers.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controller 18 | 19 | import ( 20 | "k8s.io/client-go/tools/cache" 21 | ) 22 | 23 | func HandleAddUpdateWith(f func(obj interface{})) cache.ResourceEventHandlerFuncs { 24 | return cache.ResourceEventHandlerFuncs{ 25 | AddFunc: f, 26 | UpdateFunc: func(old, new interface{}) { 27 | f(new) 28 | }, 29 | } 30 | } 31 | 32 | func HandleAllWith(f func(obj interface{})) cache.ResourceEventHandlerFuncs { 33 | return cache.ResourceEventHandlerFuncs{ 34 | AddFunc: f, 35 | UpdateFunc: func(old, new interface{}) { 36 | f(new) 37 | }, 38 | DeleteFunc: func(obj interface{}) { 39 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 40 | if ok { 41 | f(tombstone.Obj) 42 | } else { 43 | f(obj) 44 | } 45 | }, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /hack/verify-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 22 | 23 | DIFFROOT="${SCRIPT_ROOT}/pkg" 24 | TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg" 25 | _tmp="${SCRIPT_ROOT}/_tmp" 26 | 27 | cleanup() { 28 | rm -rf "${_tmp}" 29 | } 30 | trap "cleanup" EXIT SIGINT 31 | 32 | cleanup 33 | 34 | mkdir -p "${TMP_DIFFROOT}" 35 | cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" 36 | 37 | "${SCRIPT_ROOT}/hack/update-codegen.sh" 38 | echo "diffing ${DIFFROOT} against freshly generated codegen" 39 | ret=0 40 | diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? 41 | cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" 42 | if [[ $ret -eq 0 ]]; then 43 | echo "${DIFFROOT} up to date." 44 | else 45 | echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" 46 | exit 1 47 | fi 48 | -------------------------------------------------------------------------------- /hack/update-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Copyright 2023 The Multicluster-Scheduler Authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -o errexit 20 | set -o nounset 21 | set -o pipefail 22 | 23 | GOPATH="${GOPATH:-"$HOME/go"}" 24 | 25 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 26 | CODEGEN_PKG=${CODEGEN_PKG:-$( 27 | cd "${SCRIPT_ROOT}" 28 | ls -d -1 $GOPATH/pkg/mod/k8s.io/code-generator@v0.30.5 2>/dev/null || echo ../code-generator 29 | )} 30 | 31 | source "${CODEGEN_PKG}/kube_codegen.sh" 32 | 33 | kube::codegen::gen_helpers \ 34 | --boilerplate "${SCRIPT_ROOT}"/hack/boilerplate.go.txt \ 35 | "${SCRIPT_ROOT}/pkg/apis" 36 | 37 | 38 | kube::codegen::gen_client \ 39 | --with-watch \ 40 | --output-dir "${SCRIPT_ROOT}/pkg/generated" \ 41 | --output-pkg admiralty.io/multicluster-scheduler/pkg/generated \ 42 | --boilerplate "${SCRIPT_ROOT}"/hack/boilerplate.go.txt \ 43 | "${SCRIPT_ROOT}/pkg/apis" 44 | -------------------------------------------------------------------------------- /pkg/model/virtualnode/model.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package virtualnode 18 | 19 | import ( 20 | "admiralty.io/multicluster-scheduler/pkg/common" 21 | ) 22 | 23 | func BaseLabels(targetNamespace, targetName string) map[string]string { 24 | l := map[string]string{ 25 | "type": "virtual-kubelet", 26 | common.LabelAndTaintKeyVirtualKubeletProvider: common.VirtualKubeletProviderName, 27 | "kubernetes.io/role": "cluster", 28 | "alpha.service-controller.kubernetes.io/exclude-balancer": "true", 29 | "node.kubernetes.io/exclude-from-external-load-balancers": "true", 30 | } 31 | if targetNamespace == "" { 32 | l[common.LabelKeyClusterTargetName] = targetName 33 | } else { 34 | l[common.LabelKeyTargetNamespace] = targetNamespace 35 | l[common.LabelKeyTargetName] = targetName 36 | } 37 | return l 38 | } 39 | -------------------------------------------------------------------------------- /hack/version-bump.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2022 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | BEFORE_VERSION="$1" 21 | AFTER_VERSION="$2" 22 | 23 | sed_opt="-i" 24 | regex_opt="" 25 | #sed_opt="--quiet" 26 | #regex_opt="p" 27 | 28 | sed $sed_opt "s/--version $BEFORE_VERSION/--version $AFTER_VERSION/g$regex_opt" docs/quick_start.md 29 | sed $sed_opt "s/:$BEFORE_VERSION/:$AFTER_VERSION/g$regex_opt" docs/quick_start.md 30 | sed $sed_opt "s/--version $BEFORE_VERSION/--version $AFTER_VERSION/g$regex_opt" docs/operator_guide/installation.md 31 | sed $sed_opt "s/^version: $BEFORE_VERSION$/version: $AFTER_VERSION/$regex_opt" charts/multicluster-scheduler/Chart.yaml 32 | sed $sed_opt "s/^appVersion: $BEFORE_VERSION$/appVersion: $AFTER_VERSION/$regex_opt" charts/multicluster-scheduler/Chart.yaml 33 | sed $sed_opt "s/$BEFORE_VERSION/$AFTER_VERSION/g$regex_opt" charts/multicluster-scheduler/README.md 34 | -------------------------------------------------------------------------------- /test/e2e/virtual-node-labels/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2021 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | virtual-node-labels_test() { 23 | i=$1 24 | j=$2 25 | 26 | k $j label node --all kubernetes.azure.com/cluster=cluster$j --overwrite 27 | while [[ "$(k $i get node admiralty-default-c$j -o json | jq -r '.metadata.labels["kubernetes.azure.com/cluster"]')" != cluster$j ]]; do sleep 1; done 28 | k $i patch target c$j --type=merge -p '{"spec":{"excludedLabelsRegexp": "^kubernetes\\.azure\\.com/cluster="}}' 29 | while [[ "$(k $i get node admiralty-default-c$j -o json | jq -r '.metadata.labels["kubernetes.azure.com/cluster"]')" == cluster$j ]]; do sleep 1; done 30 | k $i patch target c$j --type=merge -p '{"spec":{"excludedLabelsRegexp": null}}' 31 | k $j label node --all kubernetes.azure.com/cluster- 32 | } 33 | 34 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 35 | virtual-node-labels_test "${@}" 36 | fi 37 | -------------------------------------------------------------------------------- /test/e2e/no-rogue-finalizer/test.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022 The Multicluster-Scheduler Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | set -euo pipefail 18 | 19 | source test/e2e/aliases.sh 20 | 21 | no-rogue-finalizer_test() { 22 | # give the controllers 30s to clean up after other test objects have been deleted 23 | export -f no-rogue-finalizer_test_iteration 24 | timeout --foreground 30s bash -c "until no-rogue-finalizer_test_iteration; do sleep 1; done" 25 | # use --foreground to catch ctrl-c 26 | # https://unix.stackexchange.com/a/233685 27 | } 28 | 29 | no-rogue-finalizer_test_iteration() { 30 | set -euo pipefail 31 | source test/e2e/aliases.sh 32 | 33 | # check that we didn't add finalizers to uncontrolled resources 34 | finalizer="multicluster.admiralty.io/" 35 | for resource in pods configmaps secrets services ingresses; do 36 | k 1 get $resource -A -o custom-columns=FINALIZERS:.metadata.finalizers | grep -qv $finalizer 37 | done 38 | } 39 | 40 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 41 | no-rogue-finalizer_test "${@}" 42 | fi 43 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/post-delete/job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: {{ include "fullname" . }}-post-delete-hook 5 | labels: {{ include "labels" . | nindent 4 }} 6 | annotations: 7 | "helm.sh/hook": post-delete 8 | "helm.sh/hook-weight": "1" 9 | "helm.sh/hook-delete-policy": hook-succeeded 10 | spec: 11 | template: 12 | metadata: 13 | labels: {{ include "labels" . | nindent 8 }} 14 | spec: 15 | restartPolicy: Never 16 | containers: 17 | - name: remove-finalizers 18 | image: {{ .Values.postDeleteJob.image.repository }}:{{ default .Chart.AppVersion .Values.postDeleteJob.image.tag }} 19 | imagePullPolicy: {{ .Values.postDeleteJob.image.pullPolicy }} 20 | {{- with .Values.postDeleteJob.resources }} 21 | resources: {{ toYaml . | nindent 12 }} 22 | {{- end }} 23 | serviceAccountName: {{ include "fullname" . }}-post-delete-hook 24 | {{- with .Values.imagePullSecretName }} 25 | imagePullSecrets: 26 | - name: {{ . }} 27 | {{- end }} 28 | {{- with .Values.postDeleteJob.nodeSelector }} 29 | nodeSelector: {{ toYaml . | nindent 8 }} 30 | {{- end }} 31 | {{- with .Values.postDeleteJob.securityContext }} 32 | securityContext: {{ toYaml . | nindent 8 }} 33 | {{- end }} 34 | {{- with .Values.postDeleteJob.affinity }} 35 | affinity: {{ toYaml . | nindent 8 }} 36 | {{- end }} 37 | {{- with .Values.postDeleteJob.tolerations }} 38 | tolerations: {{ toYaml . | nindent 8 }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /charts/multicluster-scheduler/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 6 | {{- end -}} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- if .Values.fullnameOverride -}} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 16 | {{- else -}} 17 | {{- $name := default .Chart.Name .Values.nameOverride -}} 18 | {{- if contains $name .Release.Name -}} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 20 | {{- else -}} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 22 | {{- end -}} 23 | {{- end -}} 24 | {{- end -}} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 31 | {{- end -}} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "labels" -}} 37 | helm.sh/chart: {{ include "chart" . }} 38 | {{ include "selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end -}} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end -}} 52 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package internalinterfaces 20 | 21 | import ( 22 | time "time" 23 | 24 | versioned "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | runtime "k8s.io/apimachinery/pkg/runtime" 27 | cache "k8s.io/client-go/tools/cache" 28 | ) 29 | 30 | // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. 31 | type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer 32 | 33 | // SharedInformerFactory a small interface to allow for adding an informer without an import cycle 34 | type SharedInformerFactory interface { 35 | Start(stopCh <-chan struct{}) 36 | InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer 37 | } 38 | 39 | // TweakListOptionsFunc is a function that transforms a v1.ListOptions. 40 | type TweakListOptionsFunc func(*v1.ListOptions) 41 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/podchaperon_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // +genclient 25 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 26 | 27 | // PodChaperon is the Schema for the podchaperons API 28 | type PodChaperon struct { 29 | metav1.TypeMeta `json:",inline"` 30 | // +optional 31 | metav1.ObjectMeta `json:"metadata,omitempty"` 32 | 33 | // +optional 34 | Spec v1.PodSpec `json:"spec,omitempty"` 35 | // +optional 36 | Status v1.PodStatus `json:"status,omitempty"` 37 | } 38 | 39 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 40 | 41 | // PodChaperonList contains a list of PodChaperon 42 | type PodChaperonList struct { 43 | metav1.TypeMeta `json:",inline"` 44 | // +optional 45 | metav1.ListMeta `json:"metadata,omitempty"` 46 | Items []PodChaperon `json:"items"` 47 | } 48 | 49 | func init() { 50 | SchemeBuilder.Register(&PodChaperon{}, &PodChaperonList{}) 51 | } 52 | -------------------------------------------------------------------------------- /release/image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | # constants 21 | default_registry="public.ecr.aws/admiralty" 22 | default_archs="amd64 arm64 ppc64le s390x" 23 | 24 | # environment variables 25 | # required 26 | IMG="${IMG}" 27 | VERSION="${VERSION}" 28 | # optional 29 | REGISTRY="${REGISTRY:-$default_registry}" 30 | ARCHS="${ARCHS:-$default_archs}" 31 | 32 | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws 33 | 34 | read -ra archs <<<"$ARCHS" 35 | 36 | arch_imgs=() 37 | for arch in "${archs[@]}"; do 38 | arch_img="$REGISTRY/$IMG:$VERSION-$arch" 39 | docker tag "$IMG:$VERSION-$arch" "$arch_img" 40 | docker push "$arch_img" 41 | arch_imgs+=("$arch_img") 42 | done 43 | 44 | export DOCKER_CLI_EXPERIMENTAL=enabled 45 | 46 | docker manifest create "$REGISTRY/$IMG:$VERSION" "${arch_imgs[@]}" 47 | for arch in "${archs[@]}"; do 48 | docker manifest annotate --arch "$arch" "$REGISTRY/$IMG:$VERSION" "$REGISTRY/$IMG:$VERSION-$arch" 49 | done 50 | docker manifest push --purge "$REGISTRY/$IMG:$VERSION" 51 | -------------------------------------------------------------------------------- /test/e2e/ingress/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2022 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | ingress_test() { 23 | i=$1 24 | j=$2 25 | 26 | k $i apply -f test/e2e/ingress/test.yaml 27 | 28 | export -f ingress_test_iteration 29 | timeout --foreground 30s bash -c "until ingress_test_iteration $i $j; do sleep 1; done" 30 | # use --foreground to catch ctrl-c 31 | # https://unix.stackexchange.com/a/233685 32 | 33 | k $i delete -f test/e2e/ingress/test.yaml 34 | } 35 | 36 | ingress_test_iteration() { 37 | i=$1 38 | j=$2 39 | 40 | set -euo pipefail 41 | source test/e2e/aliases.sh 42 | 43 | [ $(k "$j" get ingress | wc -l) -eq 2 ] || return 1 # including header 44 | [ $(k "$j" get service | wc -l) -eq 3 ] || return 1 # including header and the "kubernetes" service 45 | 46 | k "$i" annotate ing follow annotate=test --overwrite 47 | k "$j" get ing follow --no-headers -o custom-columns=ANNOTATIONS:.metadata.annotations | grep "annotate:test" 48 | } 49 | 50 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 51 | ingress_test "${@}" 52 | fi 53 | -------------------------------------------------------------------------------- /docs/operator_guide/safely_drain_cluster.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Safely Drain a Cluster 3 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/operator_guide/safely_drain_cluster.md 4 | --- 5 | 6 | Just like you can [safely drain a node](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/) with Kubernetes, you can safely drain a cluster with Admiralty. The operation even respects [PodDisruptionBudgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/). This is possible because target clusters are represented by virtual nodes in source clusters. This feature is particularly useful in a central control plane topology to perform blue/green cluster upgrades. 7 | 8 | Given a cluster connection between a source cluster and a target named `c2` in the `default` namespace, there should be a virtual node named `admiralty-default-c2` in the source cluster. To evict all proxy pods running on that node, hence delete all corresponding delegate pods in that target cluster, run: 9 | 10 | ```sh 11 | kubectl drain admiralty-default-c2 12 | ``` 13 | 14 | Note that delegate pods in that target cluster owned by other source clusters, if any, are not affected. 15 | 16 | `kubectl drain` also cordons the virtual node, so that it becomes unschedulable (by the way, `kubectl cordon` works too!). If the evicted proxy pods are owned by other objects, e.g., ReplicaSets, they are replaced by new ones that are scheduled to available virtual nodes. 17 | 18 | If you bring the target cluster back online, you need to run 19 | 20 | ```sh 21 | kubectl uncordon admiralty-default-c2 22 | ``` 23 | 24 | in the source cluster afterward to tell the Admiralty proxy scheduler that it can resume scheduling new proxy pods onto the virtual node. 25 | -------------------------------------------------------------------------------- /test/e2e/cleanup/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | source test/e2e/admiralty.sh 22 | 23 | cleanup_test() { 24 | i=$1 25 | 26 | k $i apply -f test/e2e/cleanup/test.yaml 27 | k $i rollout status deploy cleanup 28 | nodeName="$(k $i get pod -l app=cleanup -o json | jq -er '.items[0].spec.nodeName')" 29 | j="${nodeName: -1}" 30 | k $i delete target c$j 31 | 32 | export -f cleanup_test_iteration 33 | timeout --foreground 180s bash -c "until cleanup_test_iteration $i; do sleep 1; done" 34 | # use --foreground to catch ctrl-c 35 | # https://unix.stackexchange.com/a/233685 36 | 37 | admiralty_connect $i $j 38 | k $i wait --for condition=available --timeout=120s deployment admiralty-controller-manager -n admiralty 39 | k $i delete -f test/e2e/cleanup/test.yaml 40 | } 41 | 42 | cleanup_test_iteration() { 43 | i=$1 44 | 45 | set -euo pipefail 46 | source test/e2e/aliases.sh 47 | 48 | [ $(k $i get pod -l app=cleanup -o json | jq -e '.items[0].metadata.finalizers | length') -eq 0 ] || return 1 49 | } 50 | 51 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 52 | cleanup_test "${@}" 53 | fi 54 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/clustersummary_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // +genclient 25 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 26 | // +genclient:nonNamespaced 27 | 28 | // ClusterSummary is the Schema for the clustersummaries API 29 | type ClusterSummary struct { 30 | metav1.TypeMeta `json:",inline"` 31 | // +optional 32 | metav1.ObjectMeta `json:"metadata,omitempty"` 33 | 34 | // +optional 35 | Capacity v1.ResourceList `json:"capacity,omitempty"` 36 | // +optional 37 | Allocatable v1.ResourceList `json:"allocatable,omitempty"` 38 | } 39 | 40 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 41 | // +genclient:nonNamespaced 42 | 43 | // ClusterSummaryList contains a list of ClusterSummary 44 | type ClusterSummaryList struct { 45 | metav1.TypeMeta `json:",inline"` 46 | // +optional 47 | metav1.ListMeta `json:"metadata,omitempty"` 48 | Items []ClusterSummary `json:"items"` 49 | } 50 | 51 | func init() { 52 | SchemeBuilder.Register(&ClusterSummary{}, &ClusterSummaryList{}) 53 | } 54 | -------------------------------------------------------------------------------- /docs/operator_guide/installation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Installation 3 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/operator_guide/installation.md 4 | --- 5 | 6 | 7 | 8 | 1. [Install Helm v3](https://helm.sh/docs/intro/install/) on your machine if not already installed, as it is the only supported way to install the Admiralty agent at the moment. 9 | 10 | The Admiralty agent must be installed in all clusters that you want to connect. Repeat the following steps for each cluster: 11 | 12 | 1. Set your current kubeconfig and context to target the cluster: 13 | 14 | ```shell script 15 | export KUBECONFIG=changeme # if using multiple kubeconfig files 16 | kubectl config use-context changeme # if using multiple contexts 17 | ``` 18 | 19 | 1. Refer to the [cert-manager documentation](https://cert-manager.io/docs/installation/kubernetes/) to install version 1.0+, if not already installed. 20 | 21 | 1. Install the Admiralty agent with Helm v3: 22 | 23 | ```shell script 24 | helm install admiralty oci://public.ecr.aws/admiralty/admiralty \ 25 | --namespace admiralty --create-namespace \ 26 | --version 0.17.0 \ 27 | --wait 28 | ``` 29 | 30 | ## Virtual Kubelet certificate 31 | 32 | Some cloud control planes, such as [EKS](https://docs.aws.amazon.com/eks/latest/userguide/cert-signing.html) won't sign certificates for the virtual kubelet if they don't have the right CSR SignerName value, meaning that `kubernetes.io/kubelet-serving` would be rejected as a invalid SignerName. 33 | 34 | If that's the case, you can set `VKUBELET_CSR_SIGNER_NAME` env var in the `controller-manager` deployment, or set `controllerManager.certificateSignerName` value in the helm chart, which would use the correct SignerName to be signed by the control plane. 35 | 36 | In particular, on EKS, use `beta.eks.amazonaws.com/app-serving`. -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/source_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +genclient 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | 26 | // Source is the Schema for the sources API 27 | // +k8s:openapi-gen=true 28 | type Source struct { 29 | metav1.TypeMeta `json:",inline"` 30 | // +optional 31 | metav1.ObjectMeta `json:"metadata,omitempty"` 32 | 33 | // +optional 34 | Spec SourceSpec `json:"spec,omitempty"` 35 | // +optional 36 | Status SourceStatus `json:"status,omitempty"` 37 | } 38 | 39 | type SourceSpec struct { 40 | // +optional 41 | UserName string `json:"userName,omitempty"` 42 | // +optional 43 | ServiceAccountName string `json:"serviceAccountName,omitempty"` 44 | } 45 | 46 | type SourceStatus struct { 47 | } 48 | 49 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 50 | 51 | // SourceList contains a list of Source 52 | type SourceList struct { 53 | metav1.TypeMeta `json:",inline"` 54 | // +optional 55 | metav1.ListMeta `json:"metadata,omitempty"` 56 | Items []Source `json:"items"` 57 | } 58 | 59 | func init() { 60 | SchemeBuilder.Register(&Source{}, &SourceList{}) 61 | } 62 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/multicluster/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package multicluster 20 | 21 | import ( 22 | internalinterfaces "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions/internalinterfaces" 23 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions/multicluster/v1alpha1" 24 | ) 25 | 26 | // Interface provides access to each of this group's versions. 27 | type Interface interface { 28 | // V1alpha1 provides access to shared informers for resources in V1alpha1. 29 | V1alpha1() v1alpha1.Interface 30 | } 31 | 32 | type group struct { 33 | factory internalinterfaces.SharedInformerFactory 34 | namespace string 35 | tweakListOptions internalinterfaces.TweakListOptionsFunc 36 | } 37 | 38 | // New returns a new Interface. 39 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 40 | return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 41 | } 42 | 43 | // V1alpha1 returns a new v1alpha1.Interface. 44 | func (g *group) V1alpha1() v1alpha1.Interface { 45 | return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // NOTE: Boilerplate only. Ignore this file. 18 | 19 | // Package v1alpha1 contains API Schema definitions for the multicluster v1alpha1 API group 20 | // +k8s:openapi-gen=true 21 | // +k8s:deepcopy-gen=package,register 22 | // +k8s:conversion-gen=admiralty.io/multicluster-scheduler/pkg/apis/multicluster 23 | // +k8s:defaulter-gen=TypeMeta 24 | // +groupName=multicluster.admiralty.io 25 | package v1alpha1 26 | 27 | import ( 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "sigs.k8s.io/controller-runtime/pkg/scheme" 30 | ) 31 | 32 | var ( 33 | // SchemeGroupVersion is group version used to register these objects 34 | SchemeGroupVersion = schema.GroupVersion{Group: "multicluster.admiralty.io", Version: "v1alpha1"} 35 | 36 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 37 | SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} 38 | 39 | // Required by ./zz_generated.conversion.go and ./zz_generated.defaults.go 40 | localSchemeBuilder = &SchemeBuilder.SchemeBuilder 41 | 42 | // AddToScheme is required by pkg/client/... 43 | AddToScheme = SchemeBuilder.AddToScheme 44 | ) 45 | 46 | // Resource is required by pkg/client/listers/... 47 | func Resource(resource string) schema.GroupResource { 48 | return SchemeGroupVersion.WithResource(resource).GroupResource() 49 | } 50 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | [Chat with us.](https://mattermost.admiralty.io) 4 | 5 | ## DCO Sign-Off 6 | 7 | All authors retain copyright to their work. However, we ask that you certify the origin of your work. Each commit must be signed off, e.g., using the `--signoff` option to `git commit`. The `Author` and `Signed-off-by` lines of the git commit message must match. By doing this, you certify the following (from https://developercertificate.org/): 8 | 9 | ``` 10 | Developer Certificate of Origin 11 | Version 1.1 12 | 13 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 14 | 1 Letterman Drive 15 | Suite D4700 16 | San Francisco, CA, 94129 17 | 18 | Everyone is permitted to copy and distribute verbatim copies of this 19 | license document, but changing it is not allowed. 20 | 21 | 22 | Developer's Certificate of Origin 1.1 23 | 24 | By making a contribution to this project, I certify that: 25 | 26 | (a) The contribution was created in whole or in part by me and I 27 | have the right to submit it under the open source license 28 | indicated in the file; or 29 | 30 | (b) The contribution is based upon previous work that, to the best 31 | of my knowledge, is covered under an appropriate open source 32 | license and I have the right under that license to submit that 33 | work with modifications, whether created in whole or in part 34 | by me, under the same open source license (unless I am 35 | permitted to submit under a different license), as indicated 36 | in the file; or 37 | 38 | (c) The contribution was provided directly to me by some other 39 | person who certified (a), (b) or (c) and I have not modified 40 | it. 41 | 42 | (d) I understand and agree that this project and the contribution 43 | are public and that a record of the contribution (including all 44 | personal information I submit with it, including my sign-off) is 45 | maintained indefinitely and may be redistributed consistent with 46 | this project or the open source license(s) involved. 47 | ``` -------------------------------------------------------------------------------- /test/e2e/no-reservation/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | source test/e2e/admiralty.sh 22 | 23 | no-reservation_test() { 24 | # the algorithm without a candidate scheduler marks virtual nodes as unschedulable over multiple scheduling cycles, 25 | # we need to make sure that it resets those marks after going through all nodes 26 | # because conditions may have changed when it retries 27 | k 1 cordon cluster1-control-plane 28 | k 1 cordon admiralty-default-c2 29 | k 1 apply -f test/e2e/no-reservation/test.yaml 30 | 31 | export -f no-reservation_test_iteration 32 | timeout --foreground 60s bash -c "until no-reservation_test_iteration; do sleep 1; done" 33 | # use --foreground to catch ctrl-c 34 | # https://unix.stackexchange.com/a/233685 35 | 36 | k 1 uncordon cluster1-control-plane 37 | 38 | k 1 rollout status deploy no-reservation --timeout=120s 39 | 40 | k 1 delete -f test/e2e/no-reservation/test.yaml 41 | k 1 uncordon admiralty-default-c2 42 | } 43 | 44 | no-reservation_test_iteration() { 45 | set -euo pipefail 46 | source test/e2e/aliases.sh 47 | 48 | k 1 get pod -l app=no-reservation -o json | jq -e '.items[0].status.conditions[] | select(.type == "PodScheduled") | .reason == "Unschedulable"' 49 | } 50 | 51 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 52 | no-reservation_test "${@}" 53 | fi 54 | -------------------------------------------------------------------------------- /pkg/scheduler_plugins/proxy/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package proxy 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | v1 "k8s.io/api/core/v1" 24 | 25 | "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 26 | ) 27 | 28 | func TestIsPodUnschedulable(t *testing.T) { 29 | tests := []struct { 30 | name string 31 | conditionStatus []v1.PodCondition 32 | want bool 33 | }{ 34 | { 35 | name: "reason pod unschedulable", 36 | conditionStatus: []v1.PodCondition{{Status: v1.ConditionFalse, Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable}}, 37 | want: true, 38 | }, 39 | { 40 | name: "scheduling gated", 41 | conditionStatus: []v1.PodCondition{{Status: v1.ConditionFalse, Type: v1.PodScheduled, Reason: v1.PodReasonSchedulingGated}}, 42 | want: true, 43 | }, 44 | { 45 | name: "pod scheduled", 46 | conditionStatus: []v1.PodCondition{{Status: v1.ConditionTrue, Type: v1.PodScheduled}}, 47 | want: false, 48 | }, 49 | } 50 | for _, tt := range tests { 51 | t.Run(tt.name, func(t *testing.T) { 52 | podChaperon := &v1alpha1.PodChaperon{ 53 | Status: v1.PodStatus{Conditions: tt.conditionStatus}, 54 | } 55 | res := isCandidatePodUnschedulable(podChaperon) 56 | require.Equal(t, tt.want, res) 57 | }) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/vk/node/provider.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package node 18 | 19 | import ( 20 | "context" 21 | 22 | vknode "github.com/virtual-kubelet/virtual-kubelet/node" 23 | corev1 "k8s.io/api/core/v1" 24 | 25 | "admiralty.io/multicluster-scheduler/pkg/controllers/resources" 26 | ) 27 | 28 | // NodeProvider accepts a callback from virtual-kubelet's node controller, 29 | // and exposes it to our upstream resources controller, for node status updates. 30 | // If we updated node status directly, without informing virtual-kubelet, 31 | // virtual-kubelet would override our changes every minute (with node leases enabled; more often otherwise); 32 | // which would trigger a reconcile on our end, and another override on vk's end, etc. (controllers disagreeing). 33 | // When virtual-kubelet is notified of what we think the node status should be, 34 | // it keeps it in memory, and call kube-apiserver on our behalf. 35 | type NodeProvider struct { 36 | cb func(*corev1.Node) 37 | } 38 | 39 | var _ vknode.NodeProvider = &NodeProvider{} 40 | var _ resources.NodeStatusUpdater = &NodeProvider{} 41 | 42 | func (p *NodeProvider) Ping(ctx context.Context) error { 43 | return ctx.Err() 44 | } 45 | 46 | func (p *NodeProvider) NotifyNodeStatus(ctx context.Context, cb func(*corev1.Node)) { 47 | p.cb = cb 48 | } 49 | 50 | func (p *NodeProvider) UpdateNodeStatus(node *corev1.Node) { 51 | if p.cb != nil { 52 | p.cb(node) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/leaderelection/leaderelection.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package leaderelection 18 | 19 | import ( 20 | "context" 21 | "os" 22 | "strconv" 23 | "time" 24 | 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/kubernetes" 27 | "k8s.io/client-go/tools/leaderelection" 28 | "k8s.io/client-go/tools/leaderelection/resourcelock" 29 | ) 30 | 31 | func Run(ctx context.Context, ns, name string, k kubernetes.Interface, onStartedLeading func(ctx context.Context)) { 32 | id := os.Getenv("POD_NAME") 33 | if id == "" { 34 | // while remote-debugging, we may not have a pod name 35 | id = strconv.Itoa(os.Getpid()) 36 | } 37 | leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ 38 | Lock: &resourcelock.LeaseLock{ 39 | LeaseMeta: metav1.ObjectMeta{ 40 | Namespace: ns, 41 | Name: name, 42 | }, 43 | Client: k.CoordinationV1(), 44 | LockConfig: resourcelock.ResourceLockConfig{ 45 | Identity: id, 46 | EventRecorder: nil, // TODO... 47 | }, 48 | }, 49 | LeaseDuration: 15 * time.Second, 50 | RenewDeadline: 10 * time.Second, 51 | RetryPeriod: 2 * time.Second, 52 | Callbacks: leaderelection.LeaderCallbacks{ 53 | OnStartedLeading: onStartedLeading, 54 | OnStoppedLeading: func() { 55 | // TODO log 56 | }, 57 | }, 58 | WatchDog: nil, // TODO... 59 | ReleaseOnCancel: true, 60 | Name: name, 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /test/e2e/kind.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | # images built for kind v0.24.0 23 | # https://github.com/kubernetes-sigs/kind/releases/tag/v0.24.0 24 | declare -A kind_images 25 | 26 | kind_images[1.31]="kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865" 27 | kind_images[1.30]="kindest/node:v1.30.4@sha256:976ea815844d5fa93be213437e3ff5754cd599b040946b5cca43ca45c2047114" 28 | kind_images[1.29]="kindest/node:v1.29.8@sha256:d46b7aa29567e93b27f7531d258c372e829d7224b25e3fc6ffdefed12476d3aa" 29 | kind_images[1.28]="kindest/node:v1.28.13@sha256:45d319897776e11167e4698f6b14938eb4d52eb381d9e3d7a9086c16c69a8110" 30 | kind_images[1.27]="kindest/node:v1.27.16@sha256:3fd82731af34efe19cd54ea5c25e882985bafa2c9baefe14f8deab1737d9fabe" 31 | 32 | 33 | K8S_VERSION="${K8S_VERSION:-"1.30"}" 34 | 35 | kind_setup() { 36 | i=$1 37 | 38 | CLUSTER=cluster$i 39 | 40 | if ! kind get clusters | grep $CLUSTER; then 41 | kind create cluster --name $CLUSTER --wait 5m --image "${kind_images["$K8S_VERSION"]}" 42 | fi 43 | NODE_IP=$(docker inspect "${CLUSTER}-control-plane" --format "{{ .NetworkSettings.Networks.kind.IPAddress }}") 44 | kind get kubeconfig --name $CLUSTER --internal | \ 45 | sed "s/${CLUSTER}-control-plane/${NODE_IP}/g" >kubeconfig-$CLUSTER 46 | } 47 | 48 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 49 | kind_setup $1 50 | fi 51 | -------------------------------------------------------------------------------- /pkg/model/proxypod/model.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package proxypod 18 | 19 | import ( 20 | "fmt" 21 | 22 | corev1 "k8s.io/api/core/v1" 23 | "sigs.k8s.io/yaml" 24 | 25 | "admiralty.io/multicluster-scheduler/pkg/common" 26 | ) 27 | 28 | func IsProxy(pod *corev1.Pod) bool { 29 | // The scheduler name is the best indication that a pod is a proxy pod. 30 | // The elect annotation can be added in namespaces where admiralty isn't enabled; 31 | // pod would skip mutating admission webhook and be scheduled normally (not a proxy pod), 32 | // which would cause issues for controllers looking up target client from node name. 33 | // The node name is only an indicator after scheduling, and feedback needs to be able to remove finalizer even if pod wasn't scheduled; 34 | // Also, service reroute can start working earlier this way. 35 | return pod.Spec.SchedulerName == common.ProxySchedulerName 36 | } 37 | 38 | func GetSourcePod(proxyPod *corev1.Pod) (*corev1.Pod, error) { 39 | srcPodManifest, ok := proxyPod.Annotations[common.AnnotationKeySourcePodManifest] 40 | if !ok { 41 | return nil, fmt.Errorf("no source pod manifest on proxy pod") 42 | } 43 | srcPod := &corev1.Pod{} 44 | if err := yaml.Unmarshal([]byte(srcPodManifest), srcPod); err != nil { 45 | return nil, fmt.Errorf("cannot unmarshal source pod manifest: %v", err) 46 | } 47 | return srcPod, nil 48 | } 49 | 50 | func GetScheduledClusterName(proxyPod *corev1.Pod) string { 51 | return proxyPod.Spec.NodeName 52 | } 53 | -------------------------------------------------------------------------------- /pkg/vk/node/run.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package node 18 | 19 | import ( 20 | "context" 21 | 22 | "admiralty.io/multicluster-scheduler/pkg/config/agent" 23 | "github.com/virtual-kubelet/virtual-kubelet/log" 24 | "github.com/virtual-kubelet/virtual-kubelet/node" 25 | corev1 "k8s.io/api/core/v1" 26 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/client-go/kubernetes" 29 | ) 30 | 31 | func Run(ctx context.Context, t agent.Target, client kubernetes.Interface, p node.NodeProvider) error { 32 | ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{"node": t.VirtualNodeName})) 33 | 34 | leaseClient := client.CoordinationV1().Leases(corev1.NamespaceNodeLease) 35 | 36 | n := NodeFromOpts(t) 37 | nodeRunner, err := node.NewNodeController( 38 | p, 39 | n, 40 | client.CoreV1().Nodes(), 41 | node.WithNodeEnableLeaseV1(leaseClient, 0), 42 | node.WithNodeStatusUpdateErrorHandler(func(ctx context.Context, err error) error { 43 | if !k8serrors.IsNotFound(err) { 44 | return err 45 | } 46 | 47 | log.G(ctx).Debug("node not found") 48 | newNode := n.DeepCopy() 49 | newNode.ResourceVersion = "" 50 | _, err = client.CoreV1().Nodes().Create(ctx, newNode, metav1.CreateOptions{}) 51 | if err != nil { 52 | return err 53 | } 54 | log.G(ctx).Debug("created new node") 55 | return nil 56 | }), 57 | ) 58 | if err != nil { 59 | log.G(ctx).Fatal(err) 60 | } 61 | 62 | return nodeRunner.Run(ctx) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | multiclusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var scheme = runtime.NewScheme() 31 | var codecs = serializer.NewCodecFactory(scheme) 32 | 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | multiclusterv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/generated/listers/multicluster/v1alpha1/expansion_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by lister-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | // ClusterSourceListerExpansion allows custom methods to be added to 22 | // ClusterSourceLister. 23 | type ClusterSourceListerExpansion interface{} 24 | 25 | // ClusterSummaryListerExpansion allows custom methods to be added to 26 | // ClusterSummaryLister. 27 | type ClusterSummaryListerExpansion interface{} 28 | 29 | // ClusterTargetListerExpansion allows custom methods to be added to 30 | // ClusterTargetLister. 31 | type ClusterTargetListerExpansion interface{} 32 | 33 | // PodChaperonListerExpansion allows custom methods to be added to 34 | // PodChaperonLister. 35 | type PodChaperonListerExpansion interface{} 36 | 37 | // PodChaperonNamespaceListerExpansion allows custom methods to be added to 38 | // PodChaperonNamespaceLister. 39 | type PodChaperonNamespaceListerExpansion interface{} 40 | 41 | // SourceListerExpansion allows custom methods to be added to 42 | // SourceLister. 43 | type SourceListerExpansion interface{} 44 | 45 | // SourceNamespaceListerExpansion allows custom methods to be added to 46 | // SourceNamespaceLister. 47 | type SourceNamespaceListerExpansion interface{} 48 | 49 | // TargetListerExpansion allows custom methods to be added to 50 | // TargetLister. 51 | type TargetListerExpansion interface{} 52 | 53 | // TargetNamespaceListerExpansion allows custom methods to be added to 54 | // TargetNamespaceLister. 55 | type TargetNamespaceListerExpansion interface{} 56 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/target_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +genclient 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | 26 | // Target is the Schema for the targets API 27 | // +k8s:openapi-gen=true 28 | type Target struct { 29 | metav1.TypeMeta `json:",inline"` 30 | // +optional 31 | metav1.ObjectMeta `json:"metadata,omitempty"` 32 | 33 | // +optional 34 | Spec TargetSpec `json:"spec,omitempty"` 35 | // +optional 36 | Status TargetStatus `json:"status,omitempty"` 37 | } 38 | 39 | type TargetSpec struct { 40 | // +optional 41 | Self bool `json:"self,omitempty"` 42 | // +optional 43 | KubeconfigSecret *KubeconfigSecret `json:"kubeconfigSecret,omitempty"` 44 | // +optional 45 | ExcludedLabelsRegexp *string `json:"excludedLabelsRegexp,omitempty"` 46 | } 47 | 48 | type KubeconfigSecret struct { 49 | Name string `json:"name"` 50 | // +optional 51 | Key string `json:"key,omitempty"` 52 | // +optional 53 | Context string `json:"context,omitempty"` 54 | } 55 | 56 | type TargetStatus struct { 57 | } 58 | 59 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 60 | 61 | // TargetList contains a list of Target 62 | type TargetList struct { 63 | metav1.TypeMeta `json:",inline"` 64 | // +optional 65 | metav1.ListMeta `json:"metadata,omitempty"` 66 | Items []Target `json:"items"` 67 | } 68 | 69 | func init() { 70 | SchemeBuilder.Register(&Target{}, &TargetList{}) 71 | } 72 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/clustersource_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +genclient 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | // +genclient:nonNamespaced 26 | 27 | // ClusterSource is the Schema for the clustersources API 28 | // +k8s:openapi-gen=true 29 | type ClusterSource struct { 30 | metav1.TypeMeta `json:",inline"` 31 | // +optional 32 | metav1.ObjectMeta `json:"metadata,omitempty"` 33 | 34 | // +optional 35 | Spec ClusterSourceSpec `json:"spec,omitempty"` 36 | // +optional 37 | Status ClusterSourceStatus `json:"status,omitempty"` 38 | } 39 | 40 | type ClusterSourceSpec struct { 41 | // +optional 42 | UserName string `json:"userName,omitempty"` 43 | // +optional 44 | ServiceAccount *ServiceAccountReference `json:"serviceAccount,omitempty"` 45 | } 46 | 47 | type ServiceAccountReference struct { 48 | Name string `json:"name"` 49 | Namespace string `json:"namespace"` 50 | } 51 | 52 | type ClusterSourceStatus struct { 53 | } 54 | 55 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 56 | // +genclient:nonNamespaced 57 | 58 | // ClusterSourceList contains a list of ClusterSource 59 | type ClusterSourceList struct { 60 | metav1.TypeMeta `json:",inline"` 61 | // +optional 62 | metav1.ListMeta `json:"metadata,omitempty"` 63 | Items []ClusterSource `json:"items"` 64 | } 65 | 66 | func init() { 67 | SchemeBuilder.Register(&ClusterSource{}, &ClusterSourceList{}) 68 | } 69 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/multicluster/v1alpha1/fake/fake_multicluster_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned/typed/multicluster/v1alpha1" 23 | rest "k8s.io/client-go/rest" 24 | testing "k8s.io/client-go/testing" 25 | ) 26 | 27 | type FakeMulticlusterV1alpha1 struct { 28 | *testing.Fake 29 | } 30 | 31 | func (c *FakeMulticlusterV1alpha1) ClusterSources() v1alpha1.ClusterSourceInterface { 32 | return &FakeClusterSources{c} 33 | } 34 | 35 | func (c *FakeMulticlusterV1alpha1) ClusterSummaries() v1alpha1.ClusterSummaryInterface { 36 | return &FakeClusterSummaries{c} 37 | } 38 | 39 | func (c *FakeMulticlusterV1alpha1) ClusterTargets() v1alpha1.ClusterTargetInterface { 40 | return &FakeClusterTargets{c} 41 | } 42 | 43 | func (c *FakeMulticlusterV1alpha1) PodChaperons(namespace string) v1alpha1.PodChaperonInterface { 44 | return &FakePodChaperons{c, namespace} 45 | } 46 | 47 | func (c *FakeMulticlusterV1alpha1) Sources(namespace string) v1alpha1.SourceInterface { 48 | return &FakeSources{c, namespace} 49 | } 50 | 51 | func (c *FakeMulticlusterV1alpha1) Targets(namespace string) v1alpha1.TargetInterface { 52 | return &FakeTargets{c, namespace} 53 | } 54 | 55 | // RESTClient returns a RESTClient that is used to communicate 56 | // with API server by this client implementation. 57 | func (c *FakeMulticlusterV1alpha1) RESTClient() rest.Interface { 58 | var ret *rest.RESTClient 59 | return ret 60 | } 61 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package scheme 20 | 21 | import ( 22 | multiclusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var Scheme = runtime.NewScheme() 31 | var Codecs = serializer.NewCodecFactory(Scheme) 32 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | multiclusterv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(Scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/apis/multicluster/v1alpha1/clustertarget_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +genclient 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | // +genclient:nonNamespaced 26 | 27 | // ClusterTarget is the Schema for the clustertargets API 28 | // +k8s:openapi-gen=true 29 | type ClusterTarget struct { 30 | metav1.TypeMeta `json:",inline"` 31 | // +optional 32 | metav1.ObjectMeta `json:"metadata,omitempty"` 33 | 34 | // +optional 35 | Spec ClusterTargetSpec `json:"spec,omitempty"` 36 | // +optional 37 | Status ClusterTargetStatus `json:"status,omitempty"` 38 | } 39 | 40 | type ClusterTargetSpec struct { 41 | // +optional 42 | Self bool `json:"self,omitempty"` 43 | // +optional 44 | KubeconfigSecret *ClusterKubeconfigSecret `json:"kubeconfigSecret,omitempty"` 45 | // +optional 46 | ExcludedLabelsRegexp *string `json:"excludedLabelsRegexp,omitempty"` 47 | } 48 | 49 | type ClusterKubeconfigSecret struct { 50 | Namespace string `json:"namespace"` 51 | Name string `json:"name"` 52 | // +optional 53 | Key string `json:"key,omitempty"` 54 | // +optional 55 | Context string `json:"context,omitempty"` 56 | } 57 | 58 | type ClusterTargetStatus struct { 59 | } 60 | 61 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 62 | // +genclient:nonNamespaced 63 | 64 | // ClusterTargetList contains a list of ClusterTarget 65 | type ClusterTargetList struct { 66 | metav1.TypeMeta `json:",inline"` 67 | // +optional 68 | metav1.ListMeta `json:"metadata,omitempty"` 69 | Items []ClusterTarget `json:"items"` 70 | } 71 | 72 | func init() { 73 | SchemeBuilder.Register(&ClusterTarget{}, &ClusterTargetList{}) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/name/name.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package name 18 | 19 | import ( 20 | "crypto/sha256" 21 | "encoding/hex" 22 | "fmt" 23 | "strings" 24 | ) 25 | 26 | const Short int = 63 // label values, namespaces, services, ingresses, anything else? 27 | const Long int = 253 // other names 28 | 29 | func FromParts(lengthLimit int, constantIndices []int, optionalIndices []int, parts ...string) string { 30 | m := make(map[int]bool, len(constantIndices)) 31 | for _, i := range constantIndices { 32 | m[i] = true 33 | } 34 | hashRequired := false 35 | var nonEmptyParts []string 36 | for i, p := range parts { 37 | if p != "" { 38 | nonEmptyParts = append(nonEmptyParts, p) 39 | if strings.Contains(p, "-") && !m[i] && len(parts)-len(constantIndices) > 1 { 40 | hashRequired = true 41 | } 42 | } else if len(optionalIndices) > 1 { 43 | hashRequired = true 44 | } 45 | } 46 | s := strings.Join(nonEmptyParts, "-") 47 | if !hashRequired && len(s) <= lengthLimit { 48 | return s 49 | } 50 | key := strings.Join(parts, "/") // use / here because it's not allowed in parts, so join is unique (also, use empty parts here) 51 | return appendHash(lengthLimit, key, s) 52 | } 53 | 54 | func appendHash(lengthLimit int, key string, s string) string { 55 | hashLength := 10 // TODO... configure 56 | prefix := truncate(lengthLimit-hashLength-1, s) // 1 for dash between prefix and hash 57 | h := sha256.Sum256([]byte(key)) 58 | return fmt.Sprintf("%s-%s", prefix, hex.EncodeToString(h[:])[:hashLength]) 59 | } 60 | 61 | func truncate(lengthLimit int, s string) string { 62 | tooLongBy := len(s) - lengthLimit 63 | if tooLongBy > 0 { 64 | s = s[:len(s)-tooLongBy] 65 | for s[len(s)-1:] == "-" { 66 | s = s[:len(s)-1] 67 | } 68 | } 69 | return s 70 | } 71 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Introduction 3 | slug: / 4 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/introduction.md 5 | --- 6 | 7 | Admiralty is a system of Kubernetes controllers that intelligently schedules workloads across clusters. It is simple to use and simple to integrate with other tools. It enables many multi-cluster, multi-region, multi-cloud, and hybrid (simply put, global computing) use cases: 8 | 9 | 22 | 23 | In a nutshell, here's how Admiralty works: 24 | 25 | 1. Install Admiralty in each cluster that you want to federate. Configure clusters as sources and/or targets to build a centralized or decentralized topology. 26 | 1. Annotate any pod or pod template (e.g., of a Deployment, Job, or [Argo](https://argoproj.github.io/projects/argo) Workflow, among others) in any source cluster with `multicluster.admiralty.io/elect=""`. 27 | 1. Admiralty mutates the elected pods into _proxy pods_ scheduled on [virtual-kubelet](https://virtual-kubelet.io/) nodes representing target clusters, and creates _delegate pods_ in the remote clusters (actually running the containers). 28 | 1. Pod dependencies (config maps and secrets) and dependents (services and ingresses) "follow" delegate pods, i.e., they are copied as needed to target clusters. 29 | 1. A feedback loop updates the statuses and annotations of the proxy pods to reflect the statuses and annotations of the delegate pods. 30 | 1. `kubectl logs` and `kubectl exec` work as expected. 31 | 1. Integrate with Admiralty Cloud/Enterprise, [Cilium](https://cilium.io/blog/2019/03/12/clustermesh/) and other third-party solutions to enable north-south and east-west networking across clusters. 32 | 33 | :::note Open Source and Admiralty Cloud/Enterprise 34 | This documentation covers both the Admiralty open source cluster agent and Admiralty Cloud/Enterprise. Features only available with Admiralty Cloud/Enterprise are clearly marked; in that case, as much as possible, open source and commercial third-party alternatives are discussed. 35 | ::: 36 | -------------------------------------------------------------------------------- /test/e2e/ingress/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: follow 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: follow 9 | template: 10 | metadata: 11 | labels: 12 | app: follow 13 | annotations: 14 | multicluster.admiralty.io/elect: "" 15 | spec: 16 | containers: 17 | - name: hello-world 18 | image: adamgolab/hello-world 19 | env: 20 | - name: PORT 21 | value: "80" 22 | - name: WORLD 23 | valueFrom: 24 | fieldRef: 25 | fieldPath: spec.nodeName 26 | ports: 27 | - containerPort: 80 28 | --- 29 | apiVersion: apps/v1 30 | kind: Deployment 31 | metadata: 32 | name: follow-not 33 | spec: 34 | selector: 35 | matchLabels: 36 | app: follow-not 37 | template: 38 | metadata: 39 | labels: 40 | app: follow-not 41 | spec: 42 | containers: 43 | - name: hello-world 44 | image: adamgolab/hello-world 45 | env: 46 | - name: PORT 47 | value: "80" 48 | - name: WORLD 49 | valueFrom: 50 | fieldRef: 51 | fieldPath: spec.nodeName 52 | ports: 53 | - containerPort: 80 54 | --- 55 | apiVersion: v1 56 | kind: Service 57 | metadata: 58 | name: follow 59 | spec: 60 | ports: 61 | - port: 80 62 | targetPort: 80 63 | selector: 64 | app: follow 65 | --- 66 | apiVersion: v1 67 | kind: Service 68 | metadata: 69 | name: follow-not 70 | spec: 71 | ports: 72 | - port: 80 73 | targetPort: 80 74 | selector: 75 | app: follow-not 76 | --- 77 | apiVersion: networking.k8s.io/v1 78 | kind: Ingress 79 | metadata: 80 | name: follow 81 | spec: 82 | rules: 83 | - host: follow.foo.bar.baz 84 | http: 85 | paths: 86 | - path: /* 87 | pathType: Prefix 88 | backend: 89 | service: 90 | name: follow 91 | port: 92 | number: 80 93 | --- 94 | apiVersion: networking.k8s.io/v1 95 | kind: Ingress 96 | metadata: 97 | name: follow-not 98 | spec: 99 | rules: 100 | - host: follow-not.foo.bar.baz 101 | http: 102 | paths: 103 | - path: /* 104 | pathType: Prefix 105 | backend: 106 | service: 107 | name: follow-not 108 | port: 109 | number: 80 110 | -------------------------------------------------------------------------------- /test/e2e/delete-delegate/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | source test/e2e/admiralty.sh 22 | 23 | delete-delegate_test() { 24 | i=$1 25 | j=$2 26 | 27 | k $j label node --all a=b --overwrite 28 | k $i apply -f test/e2e/delete-delegate/test.yaml 29 | k $i wait pod test-delete-delegate --for=condition=PodScheduled 30 | 31 | # when the cluster connection is interrupted for more than a minute, 32 | # the delegate pod (with restart policy always) should be recreated 33 | 34 | k $i scale deploy -n admiralty admiralty-controller-manager --replicas=0 35 | uid="$(k $j get pod -l multicluster.admiralty.io/app=delete-delegate -o json | jq -er '.items[0].metadata.uid')" 36 | echo $uid 37 | k $j delete pod -l multicluster.admiralty.io/app=delete-delegate 38 | 39 | export -f delete-delegate_test_iteration 40 | timeout --foreground 120s bash -c "until delete-delegate_test_iteration $j $uid; do sleep 1; done" 41 | # use --foreground to catch ctrl-c 42 | # https://unix.stackexchange.com/a/233685 43 | k $j wait pod -l multicluster.admiralty.io/app=delete-delegate --for=condition=PodScheduled 44 | 45 | # when the cluster connection is working, the proxy pod should be deleted 46 | # to respect the invariant that pods can't resuscitate 47 | 48 | k $i scale deploy -n admiralty admiralty-controller-manager --replicas=2 49 | k $j delete pod -l multicluster.admiralty.io/app=delete-delegate --wait --timeout=30s 50 | 51 | k $i wait pod test-delete-delegate --for=delete 52 | 53 | k $j label node --all a- 54 | } 55 | 56 | delete-delegate_test_iteration() { 57 | j=$1 58 | old_uid=$2 59 | 60 | set -euo pipefail 61 | source test/e2e/aliases.sh 62 | 63 | new_uid="$(k "$j" get pod -l multicluster.admiralty.io/app=delete-delegate -o json | jq -er '.items[0].metadata.uid')" || return 1 64 | [ "$new_uid" != "$old_uid" ] || return 1 65 | } 66 | 67 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 68 | delete-delegate_test "${@}" 69 | fi 70 | -------------------------------------------------------------------------------- /pkg/generated/listers/multicluster/v1alpha1/clustersource.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by lister-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 23 | "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/client-go/tools/cache" 26 | ) 27 | 28 | // ClusterSourceLister helps list ClusterSources. 29 | // All objects returned here must be treated as read-only. 30 | type ClusterSourceLister interface { 31 | // List lists all ClusterSources in the indexer. 32 | // Objects returned here must be treated as read-only. 33 | List(selector labels.Selector) (ret []*v1alpha1.ClusterSource, err error) 34 | // Get retrieves the ClusterSource from the index for a given name. 35 | // Objects returned here must be treated as read-only. 36 | Get(name string) (*v1alpha1.ClusterSource, error) 37 | ClusterSourceListerExpansion 38 | } 39 | 40 | // clusterSourceLister implements the ClusterSourceLister interface. 41 | type clusterSourceLister struct { 42 | indexer cache.Indexer 43 | } 44 | 45 | // NewClusterSourceLister returns a new ClusterSourceLister. 46 | func NewClusterSourceLister(indexer cache.Indexer) ClusterSourceLister { 47 | return &clusterSourceLister{indexer: indexer} 48 | } 49 | 50 | // List lists all ClusterSources in the indexer. 51 | func (s *clusterSourceLister) List(selector labels.Selector) (ret []*v1alpha1.ClusterSource, err error) { 52 | err = cache.ListAll(s.indexer, selector, func(m interface{}) { 53 | ret = append(ret, m.(*v1alpha1.ClusterSource)) 54 | }) 55 | return ret, err 56 | } 57 | 58 | // Get retrieves the ClusterSource from the index for a given name. 59 | func (s *clusterSourceLister) Get(name string) (*v1alpha1.ClusterSource, error) { 60 | obj, exists, err := s.indexer.GetByKey(name) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if !exists { 65 | return nil, errors.NewNotFound(v1alpha1.Resource("clustersource"), name) 66 | } 67 | return obj.(*v1alpha1.ClusterSource), nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/generated/listers/multicluster/v1alpha1/clustertarget.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by lister-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 23 | "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/client-go/tools/cache" 26 | ) 27 | 28 | // ClusterTargetLister helps list ClusterTargets. 29 | // All objects returned here must be treated as read-only. 30 | type ClusterTargetLister interface { 31 | // List lists all ClusterTargets in the indexer. 32 | // Objects returned here must be treated as read-only. 33 | List(selector labels.Selector) (ret []*v1alpha1.ClusterTarget, err error) 34 | // Get retrieves the ClusterTarget from the index for a given name. 35 | // Objects returned here must be treated as read-only. 36 | Get(name string) (*v1alpha1.ClusterTarget, error) 37 | ClusterTargetListerExpansion 38 | } 39 | 40 | // clusterTargetLister implements the ClusterTargetLister interface. 41 | type clusterTargetLister struct { 42 | indexer cache.Indexer 43 | } 44 | 45 | // NewClusterTargetLister returns a new ClusterTargetLister. 46 | func NewClusterTargetLister(indexer cache.Indexer) ClusterTargetLister { 47 | return &clusterTargetLister{indexer: indexer} 48 | } 49 | 50 | // List lists all ClusterTargets in the indexer. 51 | func (s *clusterTargetLister) List(selector labels.Selector) (ret []*v1alpha1.ClusterTarget, err error) { 52 | err = cache.ListAll(s.indexer, selector, func(m interface{}) { 53 | ret = append(ret, m.(*v1alpha1.ClusterTarget)) 54 | }) 55 | return ret, err 56 | } 57 | 58 | // Get retrieves the ClusterTarget from the index for a given name. 59 | func (s *clusterTargetLister) Get(name string) (*v1alpha1.ClusterTarget, error) { 60 | obj, exists, err := s.indexer.GetByKey(name) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if !exists { 65 | return nil, errors.NewNotFound(v1alpha1.Resource("clustertarget"), name) 66 | } 67 | return obj.(*v1alpha1.ClusterTarget), nil 68 | } 69 | -------------------------------------------------------------------------------- /test/e2e/argo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | 22 | argo_version=3.4.5 23 | argo_manifest="https://github.com/argoproj/argo-workflows/releases/download/v$argo_version/install.yaml" 24 | argo_img="argoproj/argoexec:v$argo_version" 25 | 26 | argo_setup_once() { 27 | os=${1:-linux} 28 | arch=${2:-amd64} 29 | 30 | if 31 | out=$(./argo version) || true 32 | echo "$out" | grep "$argo_version" 33 | then 34 | echo "argo already downloaded" 35 | else 36 | echo "downloading argo" 37 | curl -Lo argo.gz "https://github.com/argoproj/argo-workflows/releases/download/v$argo_version/argo-$os-$arch.gz" 38 | gunzip -f argo.gz 39 | chmod +x argo 40 | fi 41 | 42 | # to speed up container creations (loaded by kind in argo_setup_source and argo_setup_target) 43 | docker pull "$argo_img" # may already be on host 44 | } 45 | 46 | argo_setup_source() { 47 | i=$1 48 | 49 | if ! k "$i" get ns argo; then 50 | k "$i" create ns argo 51 | fi 52 | k "$i" apply -n argo -f "$argo_manifest" 53 | 54 | k "$i" apply -f examples/argo-workflows/_service-account.yaml 55 | 56 | # speed up container creations 57 | kind load docker-image "$argo_img" --name "cluster$i" 58 | } 59 | 60 | argo_setup_target() { 61 | i=$1 62 | 63 | k "$i" apply -f examples/argo-workflows/_service-account.yaml 64 | 65 | # speed up container creations 66 | kind load docker-image "$argo_img" --name "cluster$i" 67 | } 68 | 69 | argo_test() { 70 | i=$1 71 | j=$2 72 | 73 | KUBECONFIG=kubeconfig-cluster$i ./argo submit --serviceaccount argo-workflow --wait examples/argo-workflows/blog-scenario-a-multicluster.yaml 74 | # delegate pods should be spread between both clusters 75 | [ $(k "$i" get pod -l multicluster.admiralty.io/workflow | wc -l) -gt 1 ] || exit 1 76 | [ $(k "$j" get pod -l multicluster.admiralty.io/workflow | wc -l) -gt 1 ] || exit 1 77 | KUBECONFIG=kubeconfig-cluster$i ./argo delete --all 78 | } 79 | 80 | if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then 81 | argo_test "${@}" 82 | fi 83 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: 4 | push: 5 | branches: [ master, setup-ci ] 6 | paths-ignore: 7 | - 'docs/**' 8 | pull_request: 9 | branches: [ master ] 10 | paths-ignore: 11 | - 'docs/**' 12 | 13 | jobs: 14 | 15 | build: 16 | name: Verify code gen, vet, unit test, build, and build images 17 | runs-on: ubuntu-latest 18 | steps: 19 | 20 | - name: Set up Go 1.x 21 | uses: actions/setup-go@v4 22 | with: 23 | go-version: ^1.13 24 | id: go 25 | 26 | - name: Check out code into the Go module directory 27 | uses: actions/checkout@v2 28 | 29 | - name: Get dependencies 30 | run: go get -v -t -d ./... 31 | 32 | - name: Verify code gen, go vet, go test 33 | run: ./test/test.sh 34 | 35 | - name: go build, docker build 36 | run: VERSION=$GITHUB_SHA OUTPUT_TAR_GZ=true ./build/build.sh 37 | 38 | - name: Upload images as artifact 39 | uses: actions/upload-artifact@v3 40 | with: 41 | name: images 42 | path: images 43 | 44 | e2e: 45 | name: End-to-end test 46 | needs: build 47 | runs-on: ubuntu-latest 48 | continue-on-error: ${{ matrix.experimental }} 49 | strategy: 50 | fail-fast: false 51 | matrix: 52 | k8s_version: [ "1.27", "1.28", "1.29", "1.30", "1.31" ] 53 | experimental: [ false ] 54 | # workflow succeeds even if experimental job fails, 55 | # but commit/PR check/status still appears as failure overall, 56 | # so 1.22 job is commented out for now, cf. 57 | # - https://github.community/t/continue-on-error-allow-failure-ui-indication/16773/14 58 | # - https://github.com/actions/toolkit/issues/399 59 | # include: 60 | # - k8s_version: "1.22" 61 | # experimental: true 62 | steps: 63 | 64 | - name: Check out code into the Go module directory 65 | uses: actions/checkout@v2 66 | 67 | - name: Download image artifacts 68 | uses: actions/download-artifact@v3 69 | with: 70 | name: images 71 | path: images 72 | 73 | - name: docker load 74 | run: | 75 | ./build/load.sh 76 | 77 | - name: Install kubectl, helm, kind 78 | run: ./test/e2e/install_dependencies.sh 79 | 80 | - name: End-to-end test 81 | run: VERSION=$GITHUB_SHA K8S_VERSION=${{ matrix.k8s_version }} ./test/e2e/e2e.sh 82 | 83 | - name: Archive cluster dump 84 | if: ${{ failure() }} 85 | uses: actions/upload-artifact@v3 86 | with: 87 | name: cluster-dump-${{ matrix.k8s_version }} 88 | path: cluster-dump 89 | -------------------------------------------------------------------------------- /pkg/generated/listers/multicluster/v1alpha1/clustersummary.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by lister-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 23 | "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/client-go/tools/cache" 26 | ) 27 | 28 | // ClusterSummaryLister helps list ClusterSummaries. 29 | // All objects returned here must be treated as read-only. 30 | type ClusterSummaryLister interface { 31 | // List lists all ClusterSummaries in the indexer. 32 | // Objects returned here must be treated as read-only. 33 | List(selector labels.Selector) (ret []*v1alpha1.ClusterSummary, err error) 34 | // Get retrieves the ClusterSummary from the index for a given name. 35 | // Objects returned here must be treated as read-only. 36 | Get(name string) (*v1alpha1.ClusterSummary, error) 37 | ClusterSummaryListerExpansion 38 | } 39 | 40 | // clusterSummaryLister implements the ClusterSummaryLister interface. 41 | type clusterSummaryLister struct { 42 | indexer cache.Indexer 43 | } 44 | 45 | // NewClusterSummaryLister returns a new ClusterSummaryLister. 46 | func NewClusterSummaryLister(indexer cache.Indexer) ClusterSummaryLister { 47 | return &clusterSummaryLister{indexer: indexer} 48 | } 49 | 50 | // List lists all ClusterSummaries in the indexer. 51 | func (s *clusterSummaryLister) List(selector labels.Selector) (ret []*v1alpha1.ClusterSummary, err error) { 52 | err = cache.ListAll(s.indexer, selector, func(m interface{}) { 53 | ret = append(ret, m.(*v1alpha1.ClusterSummary)) 54 | }) 55 | return ret, err 56 | } 57 | 58 | // Get retrieves the ClusterSummary from the index for a given name. 59 | func (s *clusterSummaryLister) Get(name string) (*v1alpha1.ClusterSummary, error) { 60 | obj, exists, err := s.indexer.GetByKey(name) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if !exists { 65 | return nil, errors.NewNotFound(v1alpha1.Resource("clustersummary"), name) 66 | } 67 | return obj.(*v1alpha1.ClusterSummary), nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/name/name_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package name 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func Test(t *testing.T) { 24 | cases := []struct { 25 | constantIndices []int 26 | optionalIndices []int 27 | parts []string 28 | expected string 29 | description string 30 | }{ 31 | { 32 | nil, 33 | nil, 34 | []string{"default", "foo"}, 35 | "default-foo", 36 | "no hyphen", 37 | }, 38 | { 39 | nil, 40 | nil, 41 | []string{"default", "foo-bar"}, 42 | "default-foo-bar-b1f5f88a38", 43 | "with hyphen", 44 | }, 45 | { 46 | nil, 47 | nil, 48 | []string{"default-foo", "bar"}, 49 | "default-foo-bar-ce0b03a405", 50 | "with hyphen, different hash", 51 | }, 52 | { 53 | nil, 54 | nil, 55 | []string{"verylongnamespaceverylongnamespace", "verylongnameverylongnameverylongname"}, 56 | "verylongnamespaceverylongnamespace-verylongnameveryl-79f7165434", 57 | "too long", 58 | }, 59 | { 60 | nil, 61 | nil, 62 | []string{"verylongnamespaceverylongnamespace", "verylongnamevery-longnameverylongname"}, 63 | "verylongnamespaceverylongnamespace-verylongnamevery-f098147021", // 62 characters long 64 | "remove trailing hyphen", 65 | }, 66 | { 67 | []int{0}, 68 | nil, 69 | []string{"prefix", "foo-bar"}, 70 | "prefix-foo-bar", 71 | "single variable part with hyphen", 72 | }, 73 | { 74 | nil, 75 | []int{0}, 76 | []string{"", "foo"}, 77 | "foo", 78 | "single non-optional part", 79 | }, 80 | { 81 | nil, 82 | []int{0, 1}, 83 | []string{"", "foo"}, 84 | "foo-6f64c6e626", 85 | "multiple optional parts", 86 | }, 87 | { 88 | nil, 89 | []int{0, 1}, 90 | []string{"foo", ""}, 91 | "foo-14fe48f0fb", 92 | "multiple optional parts, different hash", 93 | }, 94 | } 95 | for _, c := range cases { 96 | n := FromParts(Short, c.constantIndices, c.optionalIndices, c.parts...) 97 | if n != c.expected { 98 | t.Errorf("%s failed: %s should be %s", c.description, n, c.expected) 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /test/e2e/e2e.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | source test/e2e/aliases.sh 21 | source test/e2e/admiralty.sh 22 | source test/e2e/argo.sh 23 | source test/e2e/cert-manager.sh 24 | source test/e2e/kind.sh 25 | source test/e2e/follow/test.sh 26 | source test/e2e/logs/test.sh 27 | source test/e2e/exec/test.sh 28 | source test/e2e/ingress/test.sh 29 | source test/e2e/virtual-node-labels/test.sh 30 | source test/e2e/cleanup/test.sh 31 | source test/e2e/no-reservation/test.sh 32 | source test/e2e/webhook_ready.sh 33 | source test/e2e/no-rogue-finalizer/test.sh 34 | source test/e2e/delete-chaperon/test.sh 35 | source test/e2e/delete-delegate/test.sh 36 | 37 | argo_setup_once 38 | cert_manager_setup_once 39 | 40 | cluster_dump() { 41 | if [ $? -ne 0 ]; then 42 | k 1 cluster-info dump -A --output-directory cluster-dump/1 43 | k 2 cluster-info dump -A --output-directory cluster-dump/2 44 | fi 45 | } 46 | trap cluster_dump EXIT 47 | 48 | for i in 1 2; do 49 | kind_setup $i 50 | cert_manager_setup $i 51 | admiralty_setup $i test/e2e/values.yaml 52 | done 53 | 54 | for j in 1 2 3; do 55 | admiralty_connect 1 $j 56 | done 57 | 58 | # fix GH issue #119: self target in other namespace 59 | admiralty_connect 1 1 other 60 | 61 | argo_setup_source 1 62 | argo_setup_target 2 63 | webhook_ready 1 admiralty admiralty-controller-manager admiralty admiralty-cert 64 | 65 | # TODO simulate route controller not being able to create network routes to virtual nodes 66 | #k 1 taint nodes -l virtual-kubelet.io/provider=admiralty node.kubernetes.io/network-unavailable=:NoSchedule 67 | # unfortunately, we can't use kubectl to taint nodes with node.kubernetes.io/network-unavailable 68 | # some system defaulting admission controller overwriting the reserved taint? 69 | 70 | # fix GH issue #152: different default priority in target cluster 71 | k 2 apply -f test/e2e/priorityclass.yaml 72 | 73 | argo_test 1 2 74 | follow_test 1 2 75 | logs_test 1 2 76 | exec_test 1 2 77 | ingress_test 1 2 78 | virtual-node-labels_test 1 2 79 | cleanup_test 1 80 | delete-chaperon_test 1 81 | delete-delegate_test 1 2 82 | no-reservation_test 83 | 84 | no-rogue-finalizer_test 85 | 86 | echo "ALL SUCCEEDED" 87 | -------------------------------------------------------------------------------- /cmd/restarter/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "flag" 22 | "os" 23 | "time" 24 | 25 | "admiralty.io/multicluster-scheduler/pkg/leaderelection" 26 | "admiralty.io/multicluster-service-account/pkg/config" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | kubeinformers "k8s.io/client-go/informers" 29 | "k8s.io/client-go/kubernetes" 30 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 31 | "k8s.io/sample-controller/pkg/signals" 32 | 33 | "admiralty.io/multicluster-scheduler/pkg/controllers/target" 34 | client "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned" 35 | informers "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions" 36 | ) 37 | 38 | func main() { 39 | ctx := signals.SetupSignalHandler() 40 | 41 | cfg, ns, err := config.ConfigAndNamespaceForKubeconfigAndContext("", "") 42 | utilruntime.Must(err) 43 | 44 | k, err := kubernetes.NewForConfig(cfg) 45 | utilruntime.Must(err) 46 | 47 | customClient, err := client.NewForConfig(cfg) 48 | utilruntime.Must(err) 49 | 50 | kubeInformerFactory := kubeinformers.NewSharedInformerFactory(k, time.Second*30) 51 | customInformerFactory := informers.NewSharedInformerFactory(customClient, time.Second*30) 52 | 53 | targetCtrl := target.NewController(k, ns, 54 | os.Getenv("ADMIRALTY_CONTROLLER_MANAGER_DEPLOYMENT_NAME"), 55 | os.Getenv("ADMIRALTY_PROXY_SCHEDULER_DEPLOYMENT_NAME"), 56 | customInformerFactory.Multicluster().V1alpha1().ClusterTargets(), 57 | customInformerFactory.Multicluster().V1alpha1().Targets(), 58 | kubeInformerFactory.Core().V1().Secrets()) 59 | 60 | kubeInformerFactory.Start(ctx.Done()) 61 | customInformerFactory.Start(ctx.Done()) 62 | 63 | var leaderElect bool 64 | flag.BoolVar(&leaderElect, "leader-elect", false, "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.") 65 | flag.Parse() 66 | 67 | if leaderElect { 68 | leaderelection.Run(ctx, ns, "admiralty-restarter", k, func(ctx context.Context) { 69 | utilruntime.Must(targetCtrl.Run(ctx, 1)) 70 | }) 71 | } else { 72 | utilruntime.Must(targetCtrl.Run(ctx, 1)) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pkg/vk/node/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package node 18 | 19 | import ( 20 | "os" 21 | 22 | "admiralty.io/multicluster-scheduler/pkg/config/agent" 23 | v1 "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | 26 | "admiralty.io/multicluster-scheduler/pkg/common" 27 | "admiralty.io/multicluster-scheduler/pkg/model/virtualnode" 28 | ) 29 | 30 | func NodeFromOpts(t agent.Target) *v1.Node { 31 | node := &v1.Node{ 32 | ObjectMeta: metav1.ObjectMeta{ 33 | Name: t.VirtualNodeName, 34 | Labels: virtualnode.BaseLabels(t.Namespace, t.Name), 35 | }, 36 | Spec: v1.NodeSpec{ 37 | Taints: []v1.Taint{ 38 | { 39 | Key: common.LabelAndTaintKeyVirtualKubeletProvider, 40 | Value: common.VirtualKubeletProviderName, 41 | Effect: v1.TaintEffectNoSchedule, 42 | }, 43 | }, 44 | }, 45 | Status: v1.NodeStatus{ 46 | Conditions: []v1.NodeCondition{ 47 | { 48 | Type: v1.NodeReady, 49 | Status: v1.ConditionTrue, 50 | LastHeartbeatTime: metav1.Now(), 51 | LastTransitionTime: metav1.Now(), 52 | }, 53 | //{ 54 | // Type: v1.NodeMemoryPressure, 55 | // Status: v1.ConditionFalse, 56 | // LastHeartbeatTime: metav1.Now(), 57 | // LastTransitionTime: metav1.Now(), 58 | //}, 59 | //{ 60 | // Type: v1.NodeDiskPressure, 61 | // Status: v1.ConditionFalse, 62 | // LastHeartbeatTime: metav1.Now(), 63 | // LastTransitionTime: metav1.Now(), 64 | //}, 65 | //{ 66 | // Type: v1.NodePIDPressure, 67 | // Status: v1.ConditionFalse, 68 | // LastHeartbeatTime: metav1.Now(), 69 | // LastTransitionTime: metav1.Now(), 70 | //}, 71 | //{ 72 | // Type: v1.NodeNetworkUnavailable, 73 | // Status: v1.ConditionFalse, 74 | // LastHeartbeatTime: metav1.Now(), 75 | // LastTransitionTime: metav1.Now(), 76 | //}, 77 | }, 78 | Addresses: []v1.NodeAddress{ 79 | { 80 | Type: "InternalIP", 81 | Address: os.Getenv("VKUBELET_POD_IP"), 82 | }, 83 | }, 84 | //DaemonEndpoints: v1.NodeDaemonEndpoints{ 85 | // KubeletEndpoint: v1.DaemonEndpoint{ 86 | // Port: int32(c.ListenPort), 87 | // }, 88 | //}, 89 | }, 90 | } 91 | return node 92 | } 93 | -------------------------------------------------------------------------------- /docs/concepts/topologies.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Multi-Cluster Topologies 3 | custom_edit_url: https://github.com/admiraltyio/admiralty/edit/master/docs/concepts/cluster_topologies.md 4 | --- 5 | 6 | ## Cluster Connections 7 | 8 | Admiralty connects the control planes of Kubernetes clusters. Two clusters are said to be connected as source and target when controllers in the source cluster (e.g., run by the Admiralty agent) talk to the Kubernetes API of the target cluster. Just like controllers within a single cluster, controllers in either cluster may communicate using the target cluster's Kubernetes API as a message channel. 9 | 10 | ![](cluster_connection.svg) 11 | 12 | ## Topologies 13 | 14 | Clusters and their connections form a [directed graph](https://en.wikipedia.org/wiki/Directed_graph). In particular: 15 | 16 | - Saying that cluster A is a source of cluster B is equivalent to saying that cluster B is a target of cluster A. 17 | - Each cluster can be a source and/or a target of multiple clusters. 18 | - Connections can go both ways. 19 | - Clusters can target themselves. 20 | 21 | Here are a few typical examples. 22 | 23 | ### Central Control Plane 24 | 25 | This is perhaps the most common topology. A central cluster, often called the management cluster, targets several workload clusters. This topology is useful for many of the use cases listed in [introduction](../introduction.md). 26 | 27 | ![](central_control_plane.svg) 28 | 29 | :::note The management cluster is not on the critical path. 30 | If the Admiralty agent in the management cluster or the management cluster itself breaks or loses connectivity with workload clusters, existing pods and services in the workload clusters continue to run and serve requests—if pods are evicted in the meantime, they're even recreated locally by [PodChaperons](./scheduling.md#pod-chaperons)—so your applications stay available. However, to update applications during that time, you'd have to connect to individual clusters, so we strongly recommend having a highly available management cluster (supported by most distributions) and a disaster recovery strategy just in case (e.g. a [Velero](https://velero.io/) backup). 31 | ::: 32 | 33 | ### Cloud Bursting 34 | 35 | In this topology, a primary on-prem cluster with fixed resources targets both itself and a secondary cloud-based cluster with elastic resources. Using Kubernetes scheduling preferences (node affinities), Admiralty schedules applications to the primary cluster when resources are available; to the secondary cluster otherwise. 36 | 37 | ![](cloud_bursting.svg) 38 | 39 | ### Decentralized Federation 40 | 41 | Organizations in different administrative domains, even different trust domains, managing different clusters, may want to connect their control planes, e.g., to share resources, or to delegate operations, without having to rely on a central cluster owned by a single organization. Admiralty supports this topology, especially useful for academic research platforms, or for vendor-managed deployments. 42 | 43 | ![](decentralized_federation.svg) 44 | -------------------------------------------------------------------------------- /pkg/controllers/resources/upstream_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package resources 18 | 19 | import ( 20 | "regexp" 21 | "testing" 22 | 23 | "admiralty.io/multicluster-scheduler/pkg/config/agent" 24 | "admiralty.io/multicluster-scheduler/pkg/model/virtualnode" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func Test_upstream_reconcileLabels(t *testing.T) { 29 | clusterSummaryLabels := map[string]string{"k1": "v1", "k2": "v2", "prefix.io/k1": "v1"} 30 | addBaseLabels := func(m map[string]string) map[string]string { 31 | l := virtualnode.BaseLabels("target-namespace", "target-name") 32 | for k, v := range m { 33 | l[k] = v 34 | } 35 | return l 36 | } 37 | tests := []struct { 38 | name string 39 | excludedLabelsRegexp *regexp.Regexp 40 | want map[string]string 41 | }{{ 42 | name: "no exclusions", 43 | excludedLabelsRegexp: nil, 44 | want: addBaseLabels(map[string]string{"k1": "v1", "k2": "v2", "prefix.io/k1": "v1"}), 45 | }, { 46 | name: "exclude exact aggregated key", 47 | excludedLabelsRegexp: regexp.MustCompile(`^k1=`), 48 | want: addBaseLabels(map[string]string{"k2": "v2", "prefix.io/k1": "v1"}), 49 | }, { 50 | name: "exclude aggregated key suffix", 51 | excludedLabelsRegexp: regexp.MustCompile(`k1=`), 52 | want: addBaseLabels(map[string]string{"k2": "v2"}), 53 | }, { 54 | name: "exclude exact aggregated prefix", 55 | excludedLabelsRegexp: regexp.MustCompile(`^prefix\.io/`), 56 | want: addBaseLabels(map[string]string{"k1": "v1", "k2": "v2"}), 57 | }, { 58 | name: "exclude exact aggregated key-value pair", 59 | excludedLabelsRegexp: regexp.MustCompile(`^k1=v1$`), 60 | want: addBaseLabels(map[string]string{"k2": "v2", "prefix.io/k1": "v1"}), 61 | }, { 62 | name: "exclude two keys", 63 | excludedLabelsRegexp: regexp.MustCompile(`^k1=|^k2=`), 64 | want: addBaseLabels(map[string]string{"prefix.io/k1": "v1"}), 65 | }} 66 | for _, tt := range tests { 67 | t.Run(tt.name, func(t *testing.T) { 68 | r := upstream{ 69 | target: agent.Target{Name: "target-name", Namespace: "target-namespace"}, 70 | compiledExcludedLabelsRegexp: tt.excludedLabelsRegexp, 71 | } 72 | got := r.reconcileLabels(clusterSummaryLabels) 73 | require.Equal(t, tt.want, got) 74 | }) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pkg/controllers/feedback/controller_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package feedback 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | func TestFilterContainerStatuses(t *testing.T) { 27 | testcases := []struct { 28 | Name string 29 | In []string 30 | Has []string 31 | Expect []string 32 | }{ 33 | { 34 | Name: "empty", 35 | In: []string{}, 36 | Has: []string{}, 37 | Expect: []string{}, 38 | }, 39 | { 40 | Name: "has_all", 41 | In: []string{"a", "b", "c"}, 42 | Has: []string{"a", "b", "c"}, 43 | Expect: []string{"a", "b", "c"}, 44 | }, 45 | { 46 | Name: "has_none", 47 | In: []string{"a", "b", "c"}, 48 | Has: []string{}, 49 | Expect: []string{}, 50 | }, 51 | { 52 | Name: "has_first", 53 | In: []string{"a", "b", "c"}, 54 | Has: []string{"a"}, 55 | Expect: []string{"a"}, 56 | }, 57 | { 58 | Name: "has_last", 59 | In: []string{"a", "b", "c"}, 60 | Has: []string{"c"}, 61 | Expect: []string{"c"}, 62 | }, 63 | { 64 | Name: "missing_first", 65 | In: []string{"a", "b", "c"}, 66 | Has: []string{"b", "c"}, 67 | Expect: []string{"b", "c"}, 68 | }, 69 | { 70 | Name: "missing_last", 71 | In: []string{"a", "b", "c"}, 72 | Has: []string{"a", "b"}, 73 | Expect: []string{"a", "b"}, 74 | }, 75 | } 76 | 77 | for _, tc := range testcases { 78 | t.Run(tc.Name, func(t *testing.T) { 79 | in := []corev1.ContainerStatus{} 80 | for _, name := range tc.In { 81 | in = append(in, corev1.ContainerStatus{Name: name}) 82 | } 83 | 84 | has := func(name string) bool { 85 | for _, n := range tc.Has { 86 | if name == n { 87 | return true 88 | } 89 | } 90 | return false 91 | } 92 | 93 | out := filterContainerStatuses(in, has) 94 | outNames := []string{} 95 | for _, s := range out { 96 | outNames = append(outNames, s.Name) 97 | } 98 | if !reflect.DeepEqual(outNames, tc.Expect) { 99 | t.Fatalf("expected %v, got %v", tc.Expect, outNames) 100 | } 101 | 102 | // test if input was returned as-is or copied 103 | if len(in) > 0 && len(out) > 0 { 104 | copied := &in[0] != &out[0] 105 | if reflect.DeepEqual(tc.In, tc.Expect) { 106 | if copied { 107 | t.Fatalf("expected input to be returned as-is when no filtering occurred") 108 | } 109 | } else { 110 | if !copied { 111 | t.Fatalf("expected input to be copied when filtering occurred") 112 | } 113 | } 114 | } 115 | }) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/clientset_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | clientset "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned" 23 | multiclusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned/typed/multicluster/v1alpha1" 24 | fakemulticlusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned/typed/multicluster/v1alpha1/fake" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/watch" 27 | "k8s.io/client-go/discovery" 28 | fakediscovery "k8s.io/client-go/discovery/fake" 29 | "k8s.io/client-go/testing" 30 | ) 31 | 32 | // NewSimpleClientset returns a clientset that will respond with the provided objects. 33 | // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, 34 | // without applying any validations and/or defaults. It shouldn't be considered a replacement 35 | // for a real clientset and is mostly useful in simple unit tests. 36 | func NewSimpleClientset(objects ...runtime.Object) *Clientset { 37 | o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) 38 | for _, obj := range objects { 39 | if err := o.Add(obj); err != nil { 40 | panic(err) 41 | } 42 | } 43 | 44 | cs := &Clientset{tracker: o} 45 | cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} 46 | cs.AddReactor("*", "*", testing.ObjectReaction(o)) 47 | cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { 48 | gvr := action.GetResource() 49 | ns := action.GetNamespace() 50 | watch, err := o.Watch(gvr, ns) 51 | if err != nil { 52 | return false, nil, err 53 | } 54 | return true, watch, nil 55 | }) 56 | 57 | return cs 58 | } 59 | 60 | // Clientset implements clientset.Interface. Meant to be embedded into a 61 | // struct to get a default implementation. This makes faking out just the method 62 | // you want to test easier. 63 | type Clientset struct { 64 | testing.Fake 65 | discovery *fakediscovery.FakeDiscovery 66 | tracker testing.ObjectTracker 67 | } 68 | 69 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 70 | return c.discovery 71 | } 72 | 73 | func (c *Clientset) Tracker() testing.ObjectTracker { 74 | return c.tracker 75 | } 76 | 77 | var ( 78 | _ clientset.Interface = &Clientset{} 79 | _ testing.FakeClient = &Clientset{} 80 | ) 81 | 82 | // MulticlusterV1alpha1 retrieves the MulticlusterV1alpha1Client 83 | func (c *Clientset) MulticlusterV1alpha1() multiclusterv1alpha1.MulticlusterV1alpha1Interface { 84 | return &fakemulticlusterv1alpha1.FakeMulticlusterV1alpha1{Fake: &c.Fake} 85 | } 86 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/generic.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package externalversions 20 | 21 | import ( 22 | "fmt" 23 | 24 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | cache "k8s.io/client-go/tools/cache" 27 | ) 28 | 29 | // GenericInformer is type of SharedIndexInformer which will locate and delegate to other 30 | // sharedInformers based on type 31 | type GenericInformer interface { 32 | Informer() cache.SharedIndexInformer 33 | Lister() cache.GenericLister 34 | } 35 | 36 | type genericInformer struct { 37 | informer cache.SharedIndexInformer 38 | resource schema.GroupResource 39 | } 40 | 41 | // Informer returns the SharedIndexInformer. 42 | func (f *genericInformer) Informer() cache.SharedIndexInformer { 43 | return f.informer 44 | } 45 | 46 | // Lister returns the GenericLister. 47 | func (f *genericInformer) Lister() cache.GenericLister { 48 | return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) 49 | } 50 | 51 | // ForResource gives generic access to a shared informer of the matching type 52 | // TODO extend this to unknown resources with a client pool 53 | func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { 54 | switch resource { 55 | // Group=multicluster.admiralty.io, Version=v1alpha1 56 | case v1alpha1.SchemeGroupVersion.WithResource("clustersources"): 57 | return &genericInformer{resource: resource.GroupResource(), informer: f.Multicluster().V1alpha1().ClusterSources().Informer()}, nil 58 | case v1alpha1.SchemeGroupVersion.WithResource("clustersummaries"): 59 | return &genericInformer{resource: resource.GroupResource(), informer: f.Multicluster().V1alpha1().ClusterSummaries().Informer()}, nil 60 | case v1alpha1.SchemeGroupVersion.WithResource("clustertargets"): 61 | return &genericInformer{resource: resource.GroupResource(), informer: f.Multicluster().V1alpha1().ClusterTargets().Informer()}, nil 62 | case v1alpha1.SchemeGroupVersion.WithResource("podchaperons"): 63 | return &genericInformer{resource: resource.GroupResource(), informer: f.Multicluster().V1alpha1().PodChaperons().Informer()}, nil 64 | case v1alpha1.SchemeGroupVersion.WithResource("sources"): 65 | return &genericInformer{resource: resource.GroupResource(), informer: f.Multicluster().V1alpha1().Sources().Informer()}, nil 66 | case v1alpha1.SchemeGroupVersion.WithResource("targets"): 67 | return &genericInformer{resource: resource.GroupResource(), informer: f.Multicluster().V1alpha1().Targets().Informer()}, nil 68 | 69 | } 70 | 71 | return nil, fmt.Errorf("no informer found for %v", resource) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/multicluster/v1alpha1/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | internalinterfaces "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions/internalinterfaces" 23 | ) 24 | 25 | // Interface provides access to all the informers in this group version. 26 | type Interface interface { 27 | // ClusterSources returns a ClusterSourceInformer. 28 | ClusterSources() ClusterSourceInformer 29 | // ClusterSummaries returns a ClusterSummaryInformer. 30 | ClusterSummaries() ClusterSummaryInformer 31 | // ClusterTargets returns a ClusterTargetInformer. 32 | ClusterTargets() ClusterTargetInformer 33 | // PodChaperons returns a PodChaperonInformer. 34 | PodChaperons() PodChaperonInformer 35 | // Sources returns a SourceInformer. 36 | Sources() SourceInformer 37 | // Targets returns a TargetInformer. 38 | Targets() TargetInformer 39 | } 40 | 41 | type version struct { 42 | factory internalinterfaces.SharedInformerFactory 43 | namespace string 44 | tweakListOptions internalinterfaces.TweakListOptionsFunc 45 | } 46 | 47 | // New returns a new Interface. 48 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 49 | return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 50 | } 51 | 52 | // ClusterSources returns a ClusterSourceInformer. 53 | func (v *version) ClusterSources() ClusterSourceInformer { 54 | return &clusterSourceInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} 55 | } 56 | 57 | // ClusterSummaries returns a ClusterSummaryInformer. 58 | func (v *version) ClusterSummaries() ClusterSummaryInformer { 59 | return &clusterSummaryInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} 60 | } 61 | 62 | // ClusterTargets returns a ClusterTargetInformer. 63 | func (v *version) ClusterTargets() ClusterTargetInformer { 64 | return &clusterTargetInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} 65 | } 66 | 67 | // PodChaperons returns a PodChaperonInformer. 68 | func (v *version) PodChaperons() PodChaperonInformer { 69 | return &podChaperonInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} 70 | } 71 | 72 | // Sources returns a SourceInformer. 73 | func (v *version) Sources() SourceInformer { 74 | return &sourceInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} 75 | } 76 | 77 | // Targets returns a TargetInformer. 78 | func (v *version) Targets() TargetInformer { 79 | return &targetInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} 80 | } 81 | -------------------------------------------------------------------------------- /pkg/model/delegatepod/model_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package delegatepod 18 | 19 | import ( 20 | "errors" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestChangeLabels(t *testing.T) { 27 | tests := []struct { 28 | name string 29 | inputLabels map[string]string 30 | noPrefixLabelRegex string 31 | outputLabels map[string]string 32 | expectedErr error 33 | }{ 34 | { 35 | name: "mix of keys, no skip", 36 | inputLabels: map[string]string{ 37 | "foo.com/bar": "a", 38 | "baz.com/foo": "b", 39 | "multicluster.admiralty.io/baz": "c", 40 | "a": "d", 41 | }, 42 | noPrefixLabelRegex: "", 43 | outputLabels: map[string]string{ 44 | "multicluster.admiralty.io/bar": "a", 45 | "multicluster.admiralty.io/foo": "b", 46 | "multicluster.admiralty.io/baz": "c", 47 | "multicluster.admiralty.io/a": "d", 48 | }, 49 | }, 50 | { 51 | name: "skip multiple keys", 52 | inputLabels: map[string]string{ 53 | "foo.com/bar": "a", 54 | "baz.com/foo": "b", 55 | "multicluster.admiralty.io/foo": "c", 56 | "a": "d", 57 | }, 58 | noPrefixLabelRegex: "foo.com/bar|baz.com/foo", 59 | outputLabels: map[string]string{ 60 | "foo.com/bar": "a", 61 | "baz.com/foo": "b", 62 | "multicluster.admiralty.io/foo": "c", 63 | "multicluster.admiralty.io/a": "d", 64 | }, 65 | }, 66 | { 67 | name: "skip key/value pair", 68 | inputLabels: map[string]string{ 69 | "foo.com/bar": "a", 70 | "baz.com/foo": "b", 71 | "a": "d", 72 | }, 73 | noPrefixLabelRegex: "^a=d$", 74 | outputLabels: map[string]string{ 75 | "multicluster.admiralty.io/bar": "a", 76 | "multicluster.admiralty.io/foo": "b", 77 | "a": "d", 78 | }, 79 | }, 80 | { 81 | name: "skip value", 82 | inputLabels: map[string]string{ 83 | "foo.com/bar": "skip", 84 | }, 85 | noPrefixLabelRegex: "^.*=skip$", 86 | outputLabels: map[string]string{ 87 | "foo.com/bar": "skip", 88 | }, 89 | }, 90 | { 91 | name: "invalid regex lookahead", 92 | inputLabels: map[string]string{ 93 | "foo.com/bar": "a", 94 | }, 95 | noPrefixLabelRegex: "?!foo", 96 | outputLabels: nil, 97 | expectedErr: errors.New("invalid regex"), 98 | }, 99 | } 100 | for _, tt := range tests { 101 | t.Run(tt.name, func(t *testing.T) { 102 | out, _, err := ChangeLabels(tt.inputLabels, tt.noPrefixLabelRegex) 103 | if tt.expectedErr != nil { 104 | require.Error(t, err) 105 | } 106 | require.Equal(t, tt.outputLabels, out) 107 | }) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /test/e2e/admiralty.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2023 The Multicluster-Scheduler Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -euo pipefail 19 | 20 | VERSION="${VERSION:-dev}" 21 | 22 | source test/e2e/aliases.sh 23 | source test/e2e/kind.sh 24 | source test/e2e/webhook_ready.sh 25 | 26 | admiralty_setup() { 27 | i=$1 28 | VALUES=$2 29 | 30 | kind load docker-image admiralty-agent:$VERSION-amd64 --name cluster$i 31 | kind load docker-image admiralty-scheduler:$VERSION-amd64 --name cluster$i 32 | kind load docker-image admiralty-remove-finalizers:$VERSION-amd64 --name cluster$i 33 | kind load docker-image admiralty-restarter:$VERSION-amd64 --name cluster$i 34 | 35 | h $i upgrade --install admiralty charts/multicluster-scheduler -n admiralty --create-namespace -f $VALUES \ 36 | --set controllerManager.image.repository=admiralty-agent \ 37 | --set scheduler.image.repository=admiralty-scheduler \ 38 | --set postDeleteJob.image.repository=admiralty-remove-finalizers \ 39 | --set restarter.image.repository=admiralty-restarter \ 40 | --set controllerManager.image.tag=$VERSION-amd64 \ 41 | --set scheduler.image.tag=$VERSION-amd64 \ 42 | --set postDeleteJob.image.tag=$VERSION-amd64 \ 43 | --set restarter.image.tag=$VERSION-amd64 44 | k $i delete pod --all -n admiralty 45 | } 46 | 47 | admiralty_connect() { 48 | i=$1 49 | j=$2 50 | ns="${3:-default}" 51 | 52 | if ! k "$i" get ns other; then 53 | k "$i" create ns other 54 | fi 55 | k $i label ns $ns multicluster-scheduler=enabled --overwrite 56 | 57 | if [[ $i == $j ]]; then 58 | # if self target 59 | cat < Note: This chart was built for Helm v3. 6 | 7 | > Note: Admiralty doesn't support clusters without RBAC. There's no `rbac.create` parameter. 8 | 9 | ## Post-Delete Hook 10 | 11 | Admiralty uses [finalizers](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#finalizers) for [cross-cluster garbage collection](https://twitter.com/adrienjt/status/1199467878015066112). In particular, it adds finalizers to proxy pods, global services, and config maps and secrets mounted by multi-cluster pods. The finalizers block the deletion of those objects until Admiralty deletes their delegates in other clusters. If Admiralty stopped running, and those finalizers weren't removed, object deletions would be blocked indefinitely. Therefore, when Admiralty is uninstalled, a Kubernetes job will run as a [Helm post-delete hook](https://helm.sh/docs/topics/charts_hooks/) to clean up the finalizers. 12 | 13 | ## Parameters 14 | 15 | | Key | Type | Default | Comment | 16 | | --- | --- | --- | --- | 17 | | sourceController.enabled | boolean | `true` | disable to configure source RBAC yourself | 18 | | nameOverride | string | `""` | Override chart name in object names and labels | 19 | | fullnameOverride | string | `""` | Override chart and release names in object names | 20 | | imagePullSecretName | string | `""` | | 21 | | controllerManager.replicas | integer | `2` | | 22 | | controllerManager.image.repository | string | `"public.ecr.aws/admiralty/admiralty-agent"` | | 23 | | controllerManager.image.tag | string | `"0.17.0"` | | 24 | | controllerManager.image.pullPolicy | string | `"IfNotPresent"` | | 25 | | controllerManager.resources | object | `{}` | | 26 | | controllerManager.nodeSelector | object | `{}` | | 27 | | controllerManager.securityContext | object | `{}` | | 28 | | controllerManager.affinity | object | `{}` | | 29 | | controllerManager.tolerations | array | `[]` | | 30 | | controllerManager.certificateSignerName | string | `"kubernetes.io/kubelet-serving"` | SignerName for the virtual-kubelet certificate signing request 31 | | scheduler.replicas | integer | `2` | | 32 | | scheduler.image.repository | string | `"public.ecr.aws/admiralty/admiralty-scheduler"` | | 33 | | scheduler.image.tag | string | `"0.17.0"` | | 34 | | scheduler.image.pullPolicy | string | `"IfNotPresent"` | | 35 | | scheduler.resources | object | `{}` | | 36 | | scheduler.nodeSelector | object | `{}` | | 37 | | scheduler.securityContext | object | `{}` | | 38 | | scheduler.affinity | object | `{}` | | 39 | | scheduler.tolerations | array | `[]` | | 40 | | postDeleteJob.image.repository | string | `"public.ecr.aws/admiralty/admiralty-remove-finalizers"` | | 41 | | postDeleteJob.image.tag | string | `"0.17.0"` | | 42 | | postDeleteJob.image.pullPolicy | string | `"IfNotPresent"` | | 43 | | postDeleteJob.resources | object | `{}` | | 44 | | postDeleteJob.nodeSelector | object | `{}` | | 45 | | postDeleteJob.securityContext | object | `{}` | | 46 | | postDeleteJob.affinity | object | `{}` | | 47 | | postDeleteJob.tolerations | array | `[]` | | 48 | | restarter.replicas | integer | `2` | | 49 | | restarter.image.repository | string | `"public.ecr.aws/admiralty/admiralty-remove-finalizers"` | | 50 | | restarter.image.tag | string | `"0.17.0"` | | 51 | | restarter.image.pullPolicy | string | `"IfNotPresent"` | | 52 | | restarter.resources | object | `{}` | | 53 | | restarter.nodeSelector | object | `{}` | | 54 | | restarter.securityContext | object | `{}` | | 55 | | restarter.affinity | object | `{}` | | 56 | | restarter.tolerations | array | `[]` | | 57 | | webhook.reinvocationPolicy | string | `"Never"` | | 58 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/multicluster/v1alpha1/source.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "context" 23 | time "time" 24 | 25 | multiclusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 26 | versioned "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned" 27 | internalinterfaces "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions/internalinterfaces" 28 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/listers/multicluster/v1alpha1" 29 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | runtime "k8s.io/apimachinery/pkg/runtime" 31 | watch "k8s.io/apimachinery/pkg/watch" 32 | cache "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | // SourceInformer provides access to a shared informer and lister for 36 | // Sources. 37 | type SourceInformer interface { 38 | Informer() cache.SharedIndexInformer 39 | Lister() v1alpha1.SourceLister 40 | } 41 | 42 | type sourceInformer struct { 43 | factory internalinterfaces.SharedInformerFactory 44 | tweakListOptions internalinterfaces.TweakListOptionsFunc 45 | namespace string 46 | } 47 | 48 | // NewSourceInformer constructs a new informer for Source type. 49 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 50 | // one. This reduces memory footprint and number of connections to the server. 51 | func NewSourceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { 52 | return NewFilteredSourceInformer(client, namespace, resyncPeriod, indexers, nil) 53 | } 54 | 55 | // NewFilteredSourceInformer constructs a new informer for Source type. 56 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 57 | // one. This reduces memory footprint and number of connections to the server. 58 | func NewFilteredSourceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { 59 | return cache.NewSharedIndexInformer( 60 | &cache.ListWatch{ 61 | ListFunc: func(options v1.ListOptions) (runtime.Object, error) { 62 | if tweakListOptions != nil { 63 | tweakListOptions(&options) 64 | } 65 | return client.MulticlusterV1alpha1().Sources(namespace).List(context.TODO(), options) 66 | }, 67 | WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { 68 | if tweakListOptions != nil { 69 | tweakListOptions(&options) 70 | } 71 | return client.MulticlusterV1alpha1().Sources(namespace).Watch(context.TODO(), options) 72 | }, 73 | }, 74 | &multiclusterv1alpha1.Source{}, 75 | resyncPeriod, 76 | indexers, 77 | ) 78 | } 79 | 80 | func (f *sourceInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { 81 | return NewFilteredSourceInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) 82 | } 83 | 84 | func (f *sourceInformer) Informer() cache.SharedIndexInformer { 85 | return f.factory.InformerFor(&multiclusterv1alpha1.Source{}, f.defaultInformer) 86 | } 87 | 88 | func (f *sourceInformer) Lister() v1alpha1.SourceLister { 89 | return v1alpha1.NewSourceLister(f.Informer().GetIndexer()) 90 | } 91 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/multicluster/v1alpha1/target.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "context" 23 | time "time" 24 | 25 | multiclusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 26 | versioned "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned" 27 | internalinterfaces "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions/internalinterfaces" 28 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/listers/multicluster/v1alpha1" 29 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | runtime "k8s.io/apimachinery/pkg/runtime" 31 | watch "k8s.io/apimachinery/pkg/watch" 32 | cache "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | // TargetInformer provides access to a shared informer and lister for 36 | // Targets. 37 | type TargetInformer interface { 38 | Informer() cache.SharedIndexInformer 39 | Lister() v1alpha1.TargetLister 40 | } 41 | 42 | type targetInformer struct { 43 | factory internalinterfaces.SharedInformerFactory 44 | tweakListOptions internalinterfaces.TweakListOptionsFunc 45 | namespace string 46 | } 47 | 48 | // NewTargetInformer constructs a new informer for Target type. 49 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 50 | // one. This reduces memory footprint and number of connections to the server. 51 | func NewTargetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { 52 | return NewFilteredTargetInformer(client, namespace, resyncPeriod, indexers, nil) 53 | } 54 | 55 | // NewFilteredTargetInformer constructs a new informer for Target type. 56 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 57 | // one. This reduces memory footprint and number of connections to the server. 58 | func NewFilteredTargetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { 59 | return cache.NewSharedIndexInformer( 60 | &cache.ListWatch{ 61 | ListFunc: func(options v1.ListOptions) (runtime.Object, error) { 62 | if tweakListOptions != nil { 63 | tweakListOptions(&options) 64 | } 65 | return client.MulticlusterV1alpha1().Targets(namespace).List(context.TODO(), options) 66 | }, 67 | WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { 68 | if tweakListOptions != nil { 69 | tweakListOptions(&options) 70 | } 71 | return client.MulticlusterV1alpha1().Targets(namespace).Watch(context.TODO(), options) 72 | }, 73 | }, 74 | &multiclusterv1alpha1.Target{}, 75 | resyncPeriod, 76 | indexers, 77 | ) 78 | } 79 | 80 | func (f *targetInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { 81 | return NewFilteredTargetInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) 82 | } 83 | 84 | func (f *targetInformer) Informer() cache.SharedIndexInformer { 85 | return f.factory.InformerFor(&multiclusterv1alpha1.Target{}, f.defaultInformer) 86 | } 87 | 88 | func (f *targetInformer) Lister() v1alpha1.TargetLister { 89 | return v1alpha1.NewTargetLister(f.Informer().GetIndexer()) 90 | } 91 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/multicluster/v1alpha1/clustersource.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Multicluster-Scheduler Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "context" 23 | time "time" 24 | 25 | multiclusterv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" 26 | versioned "admiralty.io/multicluster-scheduler/pkg/generated/clientset/versioned" 27 | internalinterfaces "admiralty.io/multicluster-scheduler/pkg/generated/informers/externalversions/internalinterfaces" 28 | v1alpha1 "admiralty.io/multicluster-scheduler/pkg/generated/listers/multicluster/v1alpha1" 29 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | runtime "k8s.io/apimachinery/pkg/runtime" 31 | watch "k8s.io/apimachinery/pkg/watch" 32 | cache "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | // ClusterSourceInformer provides access to a shared informer and lister for 36 | // ClusterSources. 37 | type ClusterSourceInformer interface { 38 | Informer() cache.SharedIndexInformer 39 | Lister() v1alpha1.ClusterSourceLister 40 | } 41 | 42 | type clusterSourceInformer struct { 43 | factory internalinterfaces.SharedInformerFactory 44 | tweakListOptions internalinterfaces.TweakListOptionsFunc 45 | } 46 | 47 | // NewClusterSourceInformer constructs a new informer for ClusterSource type. 48 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 49 | // one. This reduces memory footprint and number of connections to the server. 50 | func NewClusterSourceInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { 51 | return NewFilteredClusterSourceInformer(client, resyncPeriod, indexers, nil) 52 | } 53 | 54 | // NewFilteredClusterSourceInformer constructs a new informer for ClusterSource type. 55 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 56 | // one. This reduces memory footprint and number of connections to the server. 57 | func NewFilteredClusterSourceInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { 58 | return cache.NewSharedIndexInformer( 59 | &cache.ListWatch{ 60 | ListFunc: func(options v1.ListOptions) (runtime.Object, error) { 61 | if tweakListOptions != nil { 62 | tweakListOptions(&options) 63 | } 64 | return client.MulticlusterV1alpha1().ClusterSources().List(context.TODO(), options) 65 | }, 66 | WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { 67 | if tweakListOptions != nil { 68 | tweakListOptions(&options) 69 | } 70 | return client.MulticlusterV1alpha1().ClusterSources().Watch(context.TODO(), options) 71 | }, 72 | }, 73 | &multiclusterv1alpha1.ClusterSource{}, 74 | resyncPeriod, 75 | indexers, 76 | ) 77 | } 78 | 79 | func (f *clusterSourceInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { 80 | return NewFilteredClusterSourceInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) 81 | } 82 | 83 | func (f *clusterSourceInformer) Informer() cache.SharedIndexInformer { 84 | return f.factory.InformerFor(&multiclusterv1alpha1.ClusterSource{}, f.defaultInformer) 85 | } 86 | 87 | func (f *clusterSourceInformer) Lister() v1alpha1.ClusterSourceLister { 88 | return v1alpha1.NewClusterSourceLister(f.Informer().GetIndexer()) 89 | } 90 | --------------------------------------------------------------------------------