├── deploy └── helm │ └── kubernetes │ ├── README.md │ ├── templates │ ├── selfsigning-issuer.yaml │ ├── NOTES.txt │ ├── kubeadm-scripts.yaml │ ├── kubedns-manifests.yaml │ ├── extra-manifests.yaml │ ├── konnectivity-manifests.yaml │ ├── kubeadm-config.yaml │ ├── admin-configmap.yaml │ ├── scheduler-configmap.yaml │ ├── scheduler-service.yaml │ ├── controller-manager-configmap.yaml │ ├── konnectivity-server-configmap.yaml │ ├── etcd-service.yaml │ ├── controller-manager-service.yaml │ ├── etcd-backup-persistentvolumeclaim.yaml │ ├── apiserver-service.yaml │ ├── konnectivity-server-service.yaml │ ├── kubeadm-cronjob.yaml │ ├── kubernetes-front-proxy-certs.yaml │ ├── apiserver-config.yaml │ ├── konnectivity-certs.yaml │ ├── konnectivity-server-deployment.yaml │ ├── kubeadm-job.yaml │ ├── etcd-certs.yaml │ ├── scheduler-deployment.yaml │ ├── admin-deployment.yaml │ ├── etcd-backup-cronjob.yaml │ ├── controller-manager-deployment.yaml │ ├── etcd-statefulset.yaml │ ├── _helpers.tpl │ ├── kubernetes-certs.yaml │ └── apiserver-deployment.yaml │ ├── manifests │ ├── konnectivity-agent-rbac.yaml │ ├── konnectivity-server-rbac.yaml │ ├── coredns.yaml │ └── konnectivity-agent-deployment.yaml │ ├── Chart.yaml │ ├── scripts │ └── configure-cluster.sh │ └── values.yaml ├── docs └── README.md ├── hack └── version-bump.sh ├── README.md └── LICENSE /deploy/helm/kubernetes/README.md: -------------------------------------------------------------------------------- 1 | ../../../README.md -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/selfsigning-issuer.yaml: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | --- 3 | apiVersion: cert-manager.io/v1 4 | kind: Issuer 5 | metadata: 6 | name: "{{ $fullName }}-selfsigning-issuer" 7 | spec: 8 | selfSigned: {} 9 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/konnectivity-agent-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: konnectivity-agent 5 | namespace: kube-system 6 | labels: 7 | kubernetes.io/cluster-service: "true" 8 | addonmanager.kubernetes.io/mode: Reconcile 9 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | {{- $cmd := printf "kubectl exec -n %s -ti deploy/%s-admin -- sh" .Release.Namespace $fullName -}} 3 | 1. {{ $fullName }} cluster deployed. 4 | 5 | Get shell in admin container: 6 | ┌─{{ "─" | repeat (len $cmd) }}─┐ 7 | │ {{ $cmd }} │ 8 | └─{{ "─" | repeat (len $cmd) }}─┘ 9 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/kubeadm-scripts.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.admin.enabled .Values.admin.job.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-kubeadm-scripts 8 | data: 9 | configure-cluster.sh: |+ 10 | {{- tpl (.Files.Get "scripts/configure-cluster.sh") . | nindent 4 }} 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/kubedns-manifests.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.coredns.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-coredns-manifests 8 | data: 9 | {{- if .Values.coredns.enabled }} 10 | coredns.yaml: | 11 | {{- tpl (.Files.Get "manifests/coredns.yaml") . | nindent 4 }} 12 | {{- end }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | k edit sts generic-kubernetes-etcd # update: --initial-cluster-state=existing 2 | 3 | k exec -ti pod/generic-kubernetes-etcd-0 -- etcdctl member list -w table 4 | k exec -ti pod/generic-kubernetes-etcd-0 -- etcdctl member remove 3d7220137a2218ca 5 | 6 | k exec -ti pod/generic-kubernetes-etcd-0 -- etcdctl member add generic-kubernetes-etcd-2 --peer-urls=https://generic-kubernetes-etcd-2.generic-kubernetes-etcd:2380 7 | k exec -ti pod/generic-kubernetes-etcd-0 -- etcdctl endpoint status -w table 8 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/extra-manifests.yaml: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | {{- with .Values.extraManifests }} 3 | --- 4 | apiVersion: v1 5 | kind: Secret 6 | metadata: 7 | name: {{ $fullName }}-extra-manifests 8 | data: 9 | {{- range $key, $value := . }} 10 | {{- if eq (printf "%T" $value) "string" }} 11 | {{ $key }}: {{ $value | b64enc }} 12 | {{- else }} 13 | {{ $key }}: {{ printf "%s\n" (toYaml $value) | b64enc }} 14 | {{- end }} 15 | {{- end }} 16 | {{- end }} 17 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/konnectivity-server-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: system:konnectivity-server 5 | labels: 6 | kubernetes.io/cluster-service: "true" 7 | addonmanager.kubernetes.io/mode: Reconcile 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: ClusterRole 11 | name: system:auth-delegator 12 | subjects: 13 | - apiGroup: rbac.authorization.k8s.io 14 | kind: User 15 | name: system:konnectivity-server 16 | -------------------------------------------------------------------------------- /hack/version-bump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | EC=0 3 | 4 | version=$1 5 | [ -z "$version" ] && echo "version is not specified as first argument" && exit 1 6 | 7 | echo "bumping version to $version" 8 | 9 | f=README.md 10 | sed -i "s/\(kubernetes --version\) [0-9]\+\.[0-9]\+\.[0-9]\+/\1 ${version}/" README.md 11 | git diff --exit-code "$f" && echo "$f not changed" && EC=1 12 | 13 | f=deploy/helm/kubernetes/Chart.yaml 14 | sed -i "s/\(^version:\) [0-9]\+\.[0-9]\+\.[0-9]\+/\1 ${version}/" "$f" 15 | git diff --exit-code "$f" && echo "$f not changed" && EC=1 16 | 17 | if [ "$EC" != 0 ]; then 18 | echo 19 | echo "not all files were changed!" 20 | fi 21 | exit "$EC" 22 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: kubernetes 2 | description: Production-Grade Container Scheduling and Management 3 | version: 0.13.5 4 | appVersion: 1.22.4 5 | icon: https://upload.wikimedia.org/wikipedia/commons/thumb/3/39/Kubernetes_logo_without_workmark.svg/723px-Kubernetes_logo_without_workmark.svg.png 6 | keywords: 7 | - kubernetes 8 | - go 9 | - cncf 10 | - containers 11 | home: https://github.com/kubefarm/kubernetes-in-kubernetes 12 | sources: 13 | - https://github.com/kubefarm/kubernetes-in-kubernetes 14 | - https://github.com/kubernetes/kubernetes 15 | maintainers: 16 | - name: kvaps 17 | email: kvapss@gmail.com 18 | - name: krakazyabra 19 | email: pronin.egor@gmail.com 20 | - name: mrakopes 21 | email: dave@mtfbwy.cz 22 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/konnectivity-manifests.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-konnectivity-manifests 8 | data: 9 | {{- if .Values.konnectivityServer.enabled }} 10 | konnectivity-server-rbac.yaml: | 11 | {{- tpl (.Files.Get "manifests/konnectivity-server-rbac.yaml") . | nindent 4 }} 12 | {{- end }} 13 | {{- if .Values.konnectivityAgent.enabled }} 14 | konnectivity-agent-deployment.yaml: | 15 | {{- tpl (.Files.Get "manifests/konnectivity-agent-deployment.yaml") . | nindent 4 }} 16 | konnectivity-agent-rbac.yaml: | 17 | {{- tpl (.Files.Get "manifests/konnectivity-agent-rbac.yaml") . | nindent 4 }} 18 | {{- end }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/kubeadm-config.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.admin.enabled .Values.admin.job.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-kubeadm-config 8 | data: 9 | kubeadmcfg.yaml: |+ 10 | apiVersion: kubeadm.k8s.io/v1beta3 11 | kind: ClusterConfiguration 12 | {{- if .Values.controlPlaneEndpoint }} 13 | controlPlaneEndpoint: {{ .Values.controlPlaneEndpoint }} 14 | {{- else }} 15 | controlPlaneEndpoint: {{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 16 | {{- end }} 17 | {{- with .Values.networking }} 18 | networking: 19 | dnsDomain: {{ .dnsDomain }} 20 | {{- with .podSubnet }} 21 | podSubnet: {{ . }} 22 | {{- end }} 23 | serviceSubnet: {{ .serviceSubnet }} 24 | {{- end }} 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/admin-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.admin.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-admin-conf 8 | data: 9 | admin.conf: | 10 | apiVersion: v1 11 | clusters: 12 | - cluster: 13 | certificate-authority: /pki/admin-client/ca.crt 14 | server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 15 | name: default-cluster 16 | contexts: 17 | - context: 18 | cluster: default-cluster 19 | namespace: default 20 | user: default-auth 21 | name: default-context 22 | current-context: default-context 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: default-auth 27 | user: 28 | client-certificate: /pki/admin-client/tls.crt 29 | client-key: /pki/admin-client/tls.key 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/scheduler-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.scheduler.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-scheduler-conf 8 | data: 9 | scheduler.conf: | 10 | apiVersion: v1 11 | clusters: 12 | - cluster: 13 | certificate-authority: /pki/scheduler-client/ca.crt 14 | server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 15 | name: default-cluster 16 | contexts: 17 | - context: 18 | cluster: default-cluster 19 | namespace: default 20 | user: default-auth 21 | name: default-context 22 | current-context: default-context 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: default-auth 27 | user: 28 | client-certificate: /pki/scheduler-client/tls.crt 29 | client-key: /pki/scheduler-client/tls.key 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/scheduler-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.scheduler.enabled .Values.scheduler.service.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-scheduler 8 | labels: 9 | app: {{ $fullName }}-scheduler 10 | {{- with .Values.scheduler.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.scheduler.service.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.scheduler.service.type }} 19 | {{- with .Values.scheduler.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | ports: 23 | - port: {{ .Values.scheduler.service.port }} 24 | name: client 25 | {{- with .Values.scheduler.service.nodePort }} 26 | nodePort: {{ . }} 27 | {{- end }} 28 | selector: 29 | app: {{ $fullName }}-scheduler 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/controller-manager-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.controllerManager.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-controller-manager-conf 8 | data: 9 | controller-manager.conf: | 10 | apiVersion: v1 11 | clusters: 12 | - cluster: 13 | certificate-authority: /pki/controller-manager-client/ca.crt 14 | server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 15 | name: default-cluster 16 | contexts: 17 | - context: 18 | cluster: default-cluster 19 | namespace: default 20 | user: default-auth 21 | name: default-context 22 | current-context: default-context 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: default-auth 27 | user: 28 | client-certificate: /pki/controller-manager-client/tls.crt 29 | client-key: /pki/controller-manager-client/tls.key 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/konnectivity-server-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.konnectivityServer.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ $fullName }}-konnectivity-server-conf 8 | data: 9 | konnectivity-server.conf: | 10 | apiVersion: v1 11 | clusters: 12 | - cluster: 13 | certificate-authority: /pki/konnectivity-server-client/ca.crt 14 | server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }} 15 | name: default-cluster 16 | contexts: 17 | - context: 18 | cluster: default-cluster 19 | namespace: default 20 | user: default-auth 21 | name: default-context 22 | current-context: default-context 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: default-auth 27 | user: 28 | client-certificate: /pki/konnectivity-server-client/tls.crt 29 | client-key: /pki/konnectivity-server-client/tls.key 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/etcd-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.etcd.enabled .Values.etcd.service.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-etcd 8 | labels: 9 | app: {{ $fullName }}-etcd 10 | {{- with .Values.etcd.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.etcd.service.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.etcd.service.type }} 19 | {{- with .Values.etcd.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | publishNotReadyAddresses: true 23 | clusterIP: None 24 | ports: 25 | - port: {{ .Values.etcd.service.ports.client }} 26 | name: client 27 | - port: {{ .Values.etcd.service.ports.peer }} 28 | name: peer 29 | - port: {{ .Values.etcd.service.ports.metrics }} 30 | name: metrics 31 | selector: 32 | app: {{ $fullName }}-etcd 33 | {{- end }} 34 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/controller-manager-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.controllerManager.enabled .Values.controllerManager.service.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-controller-manager 8 | labels: 9 | app: {{ $fullName }}-controller-manager 10 | {{- with .Values.controllerManager.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.controllerManager.service.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.controllerManager.service.type }} 19 | {{- with .Values.controllerManager.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | ports: 23 | - port: {{ .Values.controllerManager.service.port }} 24 | name: client 25 | {{- with .Values.controllerManager.service.nodePort }} 26 | nodePort: {{ . }} 27 | {{- end }} 28 | selector: 29 | app: {{ $fullName }}-controller-manager 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/etcd-backup-persistentvolumeclaim.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.etcd.backup.enabled (not .Values.persistence.backup.existingClaim) }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | kind: PersistentVolumeClaim 5 | apiVersion: v1 6 | metadata: 7 | name: etcd-backup-{{ $fullName }}-etcd 8 | labels: 9 | app: {{ $fullName }}-etcd 10 | {{- if .Values.persistence.backup.labels }} 11 | {{- toYaml .Values.persistence.backup.labels | nindent 4 }} 12 | {{- end }} 13 | annotations: 14 | helm.sh/resource-policy: keep 15 | {{- with .Values.persistence.backup.annotations }} 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | {{- with .Values.persistence.backup.finalizers }} 19 | finalizers: 20 | {{- toYaml . | nindent 4 }} 21 | {{- end }} 22 | spec: 23 | accessModes: 24 | {{- range .Values.persistence.backup.accessModes }} 25 | - {{ . | quote }} 26 | {{- end }} 27 | {{- if .Values.persistence.backup.storageClassName }} 28 | storageClassName: {{ .Values.persistence.backup.storageClassName }} 29 | {{- end }} 30 | resources: 31 | requests: 32 | storage: {{ .Values.persistence.backup.size | quote }} 33 | {{- end }} 34 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/apiserver-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.apiServer.enabled .Values.apiServer.service.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-apiserver 8 | labels: 9 | app: {{ $fullName }}-apiserver 10 | {{- with .Values.apiServer.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | annotations: 14 | {{- with .Values.apiServer.service.annotations }} 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.apiServer.service.type }} 19 | {{- with .Values.apiServer.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | ports: 23 | - port: {{ .Values.apiServer.service.port }} 24 | name: client 25 | {{- with .Values.apiServer.service.nodePort }} 26 | nodePort: {{ . }} 27 | {{- end }} 28 | {{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "GRPC") }} 29 | - port: {{ .Values.konnectivityServer.ports.agent }} 30 | name: agent 31 | {{- with .Values.konnectivityServer.service.nodePorts.client }} 32 | nodePort: {{ . }} 33 | {{- end }} 34 | {{- end }} 35 | selector: 36 | app: {{ $fullName }}-apiserver 37 | {{- end }} 38 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/konnectivity-server-service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | name: {{ $fullName }}-konnectivity-server 8 | labels: 9 | app: {{ $fullName }}-konnectivity-server 10 | {{- with .Values.konnectivityServer.service.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | annotations: 14 | {{- with .Values.konnectivityServer.service.annotations }} 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | type: {{ .Values.konnectivityServer.service.type }} 19 | {{- with .Values.konnectivityServer.service.loadBalancerIP }} 20 | loadBalancerIP: {{ . }} 21 | {{- end }} 22 | ports: 23 | - port: {{ .Values.konnectivityServer.ports.server }} 24 | name: server 25 | {{- with .Values.konnectivityServer.service.nodePorts.server }} 26 | nodePort: {{ . }} 27 | {{- end }} 28 | - port: {{ .Values.konnectivityServer.ports.agent }} 29 | name: agent 30 | {{- with .Values.konnectivityServer.service.nodePorts.client }} 31 | nodePort: {{ . }} 32 | {{- end }} 33 | selector: 34 | app: {{ $fullName }}-konnectivity-server 35 | {{- end }} 36 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/kubeadm-cronjob.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.admin.job.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: batch/v1 5 | kind: CronJob 6 | metadata: 7 | name: "{{ $fullName }}-kubeadm-tasks" 8 | labels: 9 | app: "{{ $fullName }}-kubeadm-tasks" 10 | {{- with .Values.admin.job.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.admin.job.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | schedule: "{{ .Values.admin.job.schedule }}" 19 | successfulJobsHistoryLimit: {{ .Values.admin.job.successfulJobsHistoryLimit }} 20 | failedJobsHistoryLimit: {{ .Values.admin.job.failedJobsHistoryLimit }} 21 | jobTemplate: 22 | metadata: 23 | labels: 24 | app: "{{ $fullName }}-kubeadm-tasks" 25 | {{- with .Values.admin.job.labels }} 26 | {{- toYaml . | nindent 4 }} 27 | {{- end }} 28 | annotations: 29 | checksum/config: {{ include (print $.Template.BasePath "/kubeadm-config.yaml") . | sha256sum }} 30 | checksum/scripts: {{ include (print $.Template.BasePath "/kubeadm-scripts.yaml") . | sha256sum }} 31 | {{- with .Values.admin.job.annotations }} 32 | {{- toYaml . | nindent 8 }} 33 | {{- end }} 34 | spec: 35 | template: 36 | {{- (include (print $.Template.BasePath "/kubeadm-job.yaml") . | fromYaml ).spec.template | toYaml | nindent 8 }} 37 | {{- end }} 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes-in-Kubernetes 2 | 3 | Deploy Kubernetes in Kubernetes using Helm 4 | 5 | ![demo](https://gist.githubusercontent.com/kvaps/3cc5d772d750f8f2d36a76d00c3342b1/raw/8596ae812e83d186d0d57d448159b05f84cb0d53/kubernetes-in-kubernetes.gif) 6 | 7 | ## Requirements 8 | 9 | * Kubernetes v1.21+ 10 | * Helm v3 11 | * cert-manager v1.0.0+ 12 | 13 | ## Quick Start 14 | 15 | ### Preparation 16 | 17 | * Install [cert-manager]. 18 | 19 | * If you running over [minikube] you might also need to install a provisioner, you can use [local-path-provisioner] for example. 20 | 21 | [cert-manager]: https://cert-manager.io/docs/installation 22 | [minikube]: https://github.com/kubernetes/minikube 23 | [local-path-provisioner]: https://github.com/rancher/local-path-provisioner#installation 24 | 25 | ### Installation 26 | 27 | ```bash 28 | helm repo add kvaps https://kvaps.github.io/charts 29 | helm install foo kvaps/kubernetes --version 0.13.5 \ 30 | --namespace foo \ 31 | --create-namespace \ 32 | --set persistence.storageClassName=local-path 33 | ``` 34 | 35 | ### Cleanup 36 | 37 | ```bash 38 | kubectl delete namespace foo 39 | ``` 40 | 41 | ## Usage 42 | 43 | Kubernetes-in-Kubernetes is just a control plane, in most cases it's useless without workers. 44 | If you're looking for a real use case, check out the following projects that implement worker nodes management: 45 | 46 | * **[Kubefarm]** - Automated Kubernetes deployment and the PXE-bootable servers farm 47 | 48 | [kubefarm]: https://github.com/kubefarm/kubefarm 49 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/kubernetes-front-proxy-certs.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.apiServer.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | {{- $certName := include "kubernetes.certname" . -}} 4 | --- 5 | apiVersion: cert-manager.io/v1 6 | kind: Certificate 7 | metadata: 8 | name: "{{ $fullName }}-pki-front-proxy-ca" 9 | spec: 10 | commonName: "{{ $certName }}-front-proxy-ca" 11 | secretName: "{{ $fullName }}-pki-front-proxy-ca" 12 | duration: 87600h # 3650d 13 | renewBefore: 8760h # 365d 14 | subject: 15 | organizations: 16 | - "{{ $fullName }}" 17 | usages: 18 | - "signing" 19 | - "key encipherment" 20 | - "cert sign" 21 | isCA: true 22 | issuerRef: 23 | name: "{{ $fullName }}-selfsigning-issuer" 24 | kind: Issuer 25 | --- 26 | apiVersion: cert-manager.io/v1 27 | kind: Issuer 28 | metadata: 29 | name: "{{ $fullName }}-front-proxy-issuer" 30 | spec: 31 | ca: 32 | secretName: "{{ $fullName }}-pki-front-proxy-ca" 33 | --- 34 | apiVersion: cert-manager.io/v1 35 | kind: Certificate 36 | metadata: 37 | name: "{{ $fullName }}-pki-front-proxy-client" 38 | spec: 39 | commonName: "{{ $certName }}-front-proxy-client" 40 | secretName: "{{ $fullName }}-pki-front-proxy-client" 41 | duration: 8760h # 365d 42 | renewBefore: 4380h # 178d 43 | subject: 44 | organizations: 45 | - "system:masters" 46 | usages: 47 | - "signing" 48 | - "key encipherment" 49 | - "client auth" 50 | issuerRef: 51 | name: "{{ $fullName }}-front-proxy-issuer" 52 | kind: Issuer 53 | {{- end }} 54 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/apiserver-config.yaml: -------------------------------------------------------------------------------- 1 | {{- $fullName := include "kubernetes.fullname" . -}} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: {{ $fullName }}-apiserver-config 7 | data: 8 | egress-selector-configuration.yaml: | 9 | apiVersion: apiserver.k8s.io/v1beta1 10 | kind: EgressSelectorConfiguration 11 | egressSelections: 12 | - name: cluster 13 | connection: 14 | {{- if and .Values.konnectivityServer.enabled }} 15 | {{- if has .Values.konnectivityServer.mode (list "HTTPConnect" "GRPC") }} 16 | proxyProtocol: {{ .Values.konnectivityServer.mode }} 17 | {{- else }} 18 | {{- fail ".Values.konnectivityServer.mode supports only \"HTTPConnect\" and \"GRPC\" values" }} 19 | {{- end }} 20 | transport: 21 | {{- if eq .Values.konnectivityServer.mode "GRPC" }} 22 | uds: 23 | udsName: /run/konnectivity-server/konnectivity-server.socket 24 | {{- else }} 25 | tcp: 26 | url: "https://{{ $fullName }}-konnectivity-server:8131" 27 | TLSConfig: 28 | caBundle: /pki/konnectivity-client/ca.crt 29 | clientKey: /pki/konnectivity-client/tls.key 30 | clientCert: /pki/konnectivity-client/tls.crt 31 | {{- end }} 32 | {{- else }} 33 | proxyProtocol: Direct 34 | {{- end }} 35 | - name: master 36 | connection: 37 | proxyProtocol: Direct 38 | - name: etcd 39 | connection: 40 | proxyProtocol: Direct 41 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/konnectivity-certs.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | {{- $certName := include "kubernetes.certname" . -}} 4 | --- 5 | apiVersion: cert-manager.io/v1 6 | kind: Certificate 7 | metadata: 8 | name: "{{ $fullName }}-pki-konnectivity-ca" 9 | spec: 10 | commonName: "{{ $certName }}-konnectivity-ca" 11 | secretName: "{{ $fullName }}-pki-konnectivity-ca" 12 | duration: 87600h # 3650d 13 | renewBefore: 8760h # 365d 14 | subject: 15 | organizations: 16 | - "{{ $fullName }}" 17 | usages: 18 | - "signing" 19 | - "key encipherment" 20 | - "cert sign" 21 | isCA: true 22 | issuerRef: 23 | name: "{{ $fullName }}-selfsigning-issuer" 24 | kind: Issuer 25 | --- 26 | apiVersion: cert-manager.io/v1 27 | kind: Issuer 28 | metadata: 29 | name: "{{ $fullName }}-konnectivity-issuer" 30 | spec: 31 | ca: 32 | secretName: "{{ $fullName }}-pki-konnectivity-ca" 33 | 34 | --- 35 | {{- $svcName1 := printf "%s-konnectivity-server" $fullName }} 36 | {{- $svcName2 := printf "%s-konnectivity-server.%s" $fullName .Release.Namespace }} 37 | {{- $svcName3 := printf "%s-konnectivity-server.%s.svc" $fullName .Release.Namespace }} 38 | apiVersion: cert-manager.io/v1 39 | kind: Certificate 40 | metadata: 41 | name: "{{ $fullName }}-pki-konnectivity-server" 42 | spec: 43 | commonName: "{{ $certName }}-konnectivity-server" 44 | secretName: "{{ $fullName }}-pki-konnectivity-server" 45 | duration: 8760h # 365d 46 | renewBefore: 4380h # 178d 47 | subject: 48 | organizations: 49 | - "{{ $fullName }}" 50 | usages: 51 | - "signing" 52 | - "key encipherment" 53 | - "server auth" 54 | dnsNames: 55 | - "{{ $svcName1 }}" 56 | - "{{ $svcName2 }}" 57 | - "{{ $svcName3 }}" 58 | - "localhost" 59 | ipAddresses: 60 | - "127.0.0.1" 61 | issuerRef: 62 | name: "{{ $fullName }}-konnectivity-issuer" 63 | kind: Issuer 64 | --- 65 | apiVersion: cert-manager.io/v1 66 | kind: Certificate 67 | metadata: 68 | name: "{{ $fullName }}-pki-konnectivity-client" 69 | spec: 70 | commonName: "{{ $certName }}-konnectivity-client" 71 | secretName: "{{ $fullName }}-pki-konnectivity-client" 72 | duration: 8760h # 365d 73 | renewBefore: 4380h # 178d 74 | subject: 75 | organizations: 76 | - "system:masters" 77 | usages: 78 | - "signing" 79 | - "key encipherment" 80 | - "client auth" 81 | issuerRef: 82 | name: "{{ $fullName }}-konnectivity-issuer" 83 | kind: Issuer 84 | {{- end }} 85 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/konnectivity-server-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: "{{ $fullName }}-konnectivity-server" 8 | labels: 9 | app: "{{ $fullName }}-konnectivity-server" 10 | {{- with .Values.konnectivityServer.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.konnectivityServer.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | replicas: {{ .Values.konnectivityServer.replicaCount }} 19 | selector: 20 | matchLabels: 21 | app: "{{ $fullName }}-konnectivity-server" 22 | template: 23 | metadata: 24 | labels: 25 | app: "{{ $fullName }}-konnectivity-server" 26 | {{- with .Values.konnectivityServer.podLabels }} 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- with .Values.konnectivityServer.podAnnotations }} 30 | annotations: 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with .Values.konnectivityServer.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.konnectivityServer.tolerations }} 39 | tolerations: 40 | {{- toYaml . | nindent 6 }} 41 | {{- end }} 42 | {{- if or .Values.konnectivityServer.affinity .Values.konnectivityServer.podAntiAffinity }} 43 | affinity: 44 | {{- with .Values.konnectivityServer.affinity }} 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | {{- if eq .Values.konnectivityServer.podAntiAffinity "hard" }} 48 | podAntiAffinity: 49 | requiredDuringSchedulingIgnoredDuringExecution: 50 | - topologyKey: "{{ .Values.konnectivityServer.podAntiAffinityTopologyKey }}" 51 | labelSelector: 52 | matchLabels: 53 | app: {{ $fullName }}-konnectivity-server 54 | {{- else if eq .Values.konnectivityServer.podAntiAffinity "soft" }} 55 | podAntiAffinity: 56 | preferredDuringSchedulingIgnoredDuringExecution: 57 | - weight: 1 58 | podAffinityTerm: 59 | topologyKey: "{{ .Values.konnectivityServer.podAntiAffinityTopologyKey }}" 60 | labelSelector: 61 | matchLabels: 62 | app: {{ $fullName }}-konnectivity-server 63 | {{- end }} 64 | {{- end }} 65 | {{- with .Values.konnectivityServer.image.pullSecrets }} 66 | imagePullSecrets: 67 | {{- toYaml . | nindent 10 }} 68 | {{- end }} 69 | automountServiceAccountToken: false 70 | containers: 71 | {{ template "kubernetes.konnectivityServer.containers" . }} 72 | {{- with .Values.konnectivityServer.sidecars }} 73 | {{- toYaml . | nindent 6 }} 74 | {{- end }} 75 | securityContext: 76 | seccompProfile: 77 | type: RuntimeDefault 78 | {{ template "kubernetes.konnectivityServer.volumes" . }} 79 | {{- end }} 80 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/scripts/configure-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | set -x 4 | ENDPOINT=$(awk -F'[ "]+' '$1 == "controlPlaneEndpoint:" {print $2}' /config/kubeadmcfg.yaml) 5 | 6 | # ------------------------------------------------------------------------------ 7 | # Update secrets and component configs 8 | # ------------------------------------------------------------------------------ 9 | 10 | # wait for cluster 11 | echo "Waiting for api-server endpoint ${ENDPOINT}..." 12 | until kubectl cluster-info >/dev/null 2>/dev/null; do 13 | sleep 1 14 | done 15 | 16 | # ------------------------------------------------------------------------------ 17 | # Cluster configuration 18 | # ------------------------------------------------------------------------------ 19 | export KUBECONFIG=/etc/kubernetes/admin.conf 20 | 21 | # upload configuration 22 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/6 23 | kubeadm init phase upload-config kubeadm --config /config/kubeadmcfg.yaml 24 | 25 | # upload configuration 26 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/5 27 | kubeadm init phase upload-config kubelet --config /config/kubeadmcfg.yaml -v1 2>&1 | 28 | while read line; do echo "$line" | grep 'Preserving the CRISocket information for the control-plane node' && killall kubeadm || echo "$line"; done 29 | 30 | # setup bootstrap-tokens 31 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/7 32 | # TODO: https://github.com/kubernetes/kubernetes/issues/98881 33 | flatconfig=$(mktemp) 34 | kubectl config view --flatten > "$flatconfig" 35 | kubeadm init phase bootstrap-token --config /config/kubeadmcfg.yaml --skip-token-print --kubeconfig="$flatconfig" 36 | rm -f "$flatconfig" 37 | 38 | # correct apiserver address for the external clients 39 | kubectl apply -n kube-public -f - </dev/null || true 64 | {{- end }} 65 | 66 | {{- if .Values.konnectivityAgent.enabled }}{{"\n"}} 67 | # install konnectivity agent 68 | kubectl apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml 69 | {{- else }}{{"\n"}} 70 | # uninstall konnectivity agent 71 | kubectl delete -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml 2>/dev/null || true 72 | {{- end }} 73 | 74 | {{- if .Values.coredns.enabled }}{{"\n"}} 75 | # install coredns addon 76 | kubectl apply -f /manifests/coredns.yaml 77 | {{- else }}{{"\n"}} 78 | # uninstall coredns addon 79 | kubectl delete -f /manifests/coredns.yaml 2>/dev/null || true 80 | {{- end }} 81 | 82 | {{- if .Values.kubeProxy.enabled }}{{"\n"}} 83 | # install kube-proxy addon 84 | # TODO: https://github.com/kubefarm/kubernetes-in-kubernetes/issues/4 85 | kubeadm init phase addon kube-proxy --config /config/kubeadmcfg.yaml 86 | {{- else }}{{"\n"}} 87 | # uninstall kube-proxy addon 88 | kubectl -n kube-system delete configmap/kube-proxy daemonset/kube-proxy 2>/dev/null || true 89 | {{- end }} 90 | 91 | {{- with .Values.extraManifests }}{{"\n"}} 92 | kubectl apply{{- range $key, $value := . }} -f /manifests/{{ $key }}{{- end }} 93 | {{- end }} 94 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/kubeadm-job.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.admin.job.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: batch/v1 5 | kind: Job 6 | metadata: 7 | name: "{{ $fullName }}-kubeadm-tasks" 8 | labels: 9 | app: "{{ $fullName }}-kubeadm-tasks" 10 | {{- with .Values.admin.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | annotations: 14 | helm.sh/hook: post-install,post-upgrade 15 | helm.sh/hook-delete-policy: before-hook-creation 16 | checksum/config: {{ include (print $.Template.BasePath "/kubeadm-config.yaml") . | sha256sum }} 17 | checksum/scripts: {{ include (print $.Template.BasePath "/kubeadm-scripts.yaml") . | sha256sum }} 18 | {{- with .Values.admin.annotations }} 19 | {{- toYaml . | nindent 4 }} 20 | {{- end }} 21 | spec: 22 | template: 23 | metadata: 24 | labels: 25 | app: "{{ $fullName }}-kubeadm-tasks" 26 | {{- with .Values.admin.podLabels }} 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- with .Values.admin.podAnnotations }} 30 | annotations: 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with .Values.admin.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.admin.tolerations }} 39 | tolerations: 40 | {{- toYaml . | nindent 6 }} 41 | {{- end }} 42 | {{- with .Values.admin.affinity }} 43 | affinity: 44 | {{- toYaml . | nindent 8 }} 45 | {{- end }} 46 | {{- with .Values.admin.image.pullSecrets }} 47 | imagePullSecrets: 48 | {{- toYaml . | nindent 10 }} 49 | {{- end }} 50 | automountServiceAccountToken: false 51 | restartPolicy: OnFailure 52 | containers: 53 | - name: kubeadm 54 | {{- with .Values.admin.image }} 55 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 56 | imagePullPolicy: {{ .pullPolicy }} 57 | {{- end }} 58 | command: [ '/scripts/configure-cluster.sh' ] 59 | env: 60 | - name: KUBECONFIG 61 | value: "/etc/kubernetes/admin.conf" 62 | {{- with .Values.admin.extraEnv }} 63 | {{- toYaml . | nindent 8 }} 64 | {{- end }} 65 | volumeMounts: 66 | - mountPath: /etc/kubernetes/ 67 | name: kubeconfig 68 | readOnly: true 69 | - mountPath: /pki/admin-client 70 | name: pki-admin-client 71 | - mountPath: /scripts 72 | name: scripts 73 | {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled .Values.coredns.enabled }} 74 | - mountPath: /manifests 75 | name: manifests 76 | {{- end }} 77 | - mountPath: /config 78 | name: config 79 | {{- with .Values.admin.extraVolumeMounts }} 80 | {{- toYaml . | nindent 8 }} 81 | {{- end }} 82 | {{- with .Values.admin.sidecars }} 83 | {{- toYaml . | nindent 6 }} 84 | {{- end }} 85 | volumes: 86 | - configMap: 87 | name: "{{ $fullName }}-admin-conf" 88 | name: kubeconfig 89 | - secret: 90 | secretName: "{{ $fullName }}-pki-admin-client" 91 | name: pki-admin-client 92 | - name: scripts 93 | configMap: 94 | name: "{{ $fullName }}-kubeadm-scripts" 95 | defaultMode: 0777 96 | {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled .Values.coredns.enabled }} 97 | - name: manifests 98 | projected: 99 | sources: 100 | {{- if or .Values.extraManifests }} 101 | - secret: 102 | name: "{{ $fullName }}-extra-manifests" 103 | {{- end }} 104 | {{- if or .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 105 | - configMap: 106 | name: "{{ $fullName }}-konnectivity-manifests" 107 | {{- end }} 108 | {{- if .Values.coredns.enabled }} 109 | - configMap: 110 | name: "{{ $fullName }}-coredns-manifests" 111 | {{- end }} 112 | {{- end }} 113 | - name: config 114 | configMap: 115 | name: "{{ $fullName }}-kubeadm-config" 116 | {{- with .Values.admin.extraVolumes }} 117 | {{- toYaml . | nindent 6 }} 118 | {{- end }} 119 | {{- end }} 120 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/etcd-certs.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.etcd.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | {{- $certName := include "kubernetes.certname" . -}} 4 | --- 5 | apiVersion: cert-manager.io/v1 6 | kind: Certificate 7 | metadata: 8 | name: "{{ $fullName }}-pki-etcd-ca" 9 | spec: 10 | commonName: "{{ $certName }}-etcd-ca" 11 | secretName: "{{ $fullName }}-pki-etcd-ca" 12 | duration: 87600h # 3650d 13 | renewBefore: 8760h # 365d 14 | subject: 15 | organizations: 16 | - "{{ $fullName }}" 17 | usages: 18 | - "signing" 19 | - "key encipherment" 20 | - "cert sign" 21 | isCA: true 22 | issuerRef: 23 | name: "{{ $fullName }}-selfsigning-issuer" 24 | kind: Issuer 25 | --- 26 | apiVersion: cert-manager.io/v1 27 | kind: Issuer 28 | metadata: 29 | name: "{{ $fullName }}-etcd-issuer" 30 | spec: 31 | ca: 32 | secretName: "{{ $fullName }}-pki-etcd-ca" 33 | --- 34 | {{- $svcName1 := printf "%s-etcd" $fullName }} 35 | {{- $svcName2 := printf "%s-etcd.%s" $fullName .Release.Namespace }} 36 | {{- $svcName3 := printf "%s-etcd.%s.svc" $fullName .Release.Namespace }} 37 | {{- $podName1 := printf "*.%s-etcd" $fullName }} 38 | {{- $podName2 := printf "*.%s-etcd.%s" $fullName .Release.Namespace }} 39 | {{- $podName3 := printf "*.%s-etcd.%s.svc" $fullName .Release.Namespace }} 40 | {{- $svcClientName1 := printf "%s-etcd-client" $fullName }} 41 | {{- $svcClientName2 := printf "%s-etcd-client.%s" $fullName .Release.Namespace }} 42 | {{- $svcClientName3 := printf "%s-etcd-client.%s.svc" $fullName .Release.Namespace }} 43 | apiVersion: cert-manager.io/v1 44 | kind: Certificate 45 | metadata: 46 | name: "{{ $fullName }}-pki-etcd-peer" 47 | spec: 48 | commonName: "{{ $certName }}-etcd-peer" 49 | secretName: "{{ $fullName }}-pki-etcd-peer" 50 | duration: 8760h # 365d 51 | renewBefore: 4380h # 178d 52 | subject: 53 | organizations: 54 | - "{{ $fullName }}" 55 | usages: 56 | - "signing" 57 | - "key encipherment" 58 | - "server auth" 59 | - "client auth" 60 | dnsNames: 61 | - "{{ $svcName1 }}" 62 | - "{{ $svcName2 }}" 63 | - "{{ $svcName3 }}" 64 | - "{{ $podName1 }}" 65 | - "{{ $podName2 }}" 66 | - "{{ $podName3 }}" 67 | - "localhost" 68 | ipAddresses: 69 | - "127.0.0.1" 70 | issuerRef: 71 | name: "{{ $fullName }}-etcd-issuer" 72 | kind: Issuer 73 | --- 74 | apiVersion: cert-manager.io/v1 75 | kind: Certificate 76 | metadata: 77 | name: "{{ $fullName }}-pki-etcd-server" 78 | spec: 79 | commonName: "{{ $certName }}-etcd-server" 80 | secretName: "{{ $fullName }}-pki-etcd-server" 81 | duration: 8760h # 365d 82 | renewBefore: 4380h # 178d 83 | subject: 84 | organizations: 85 | - "{{ $fullName }}" 86 | usages: 87 | - "signing" 88 | - "key encipherment" 89 | - "server auth" 90 | - "client auth" 91 | dnsNames: 92 | - "{{ $svcName1 }}" 93 | - "{{ $svcName2 }}" 94 | - "{{ $svcName3 }}" 95 | - "{{ $podName1 }}" 96 | - "{{ $podName2 }}" 97 | - "{{ $podName3 }}" 98 | - "{{ $svcClientName1 }}" 99 | - "{{ $svcClientName2 }}" 100 | - "{{ $svcClientName3 }}" 101 | - "localhost" 102 | {{- with .Values.etcd.certSANs.dnsNames }} 103 | {{- . | toYaml | nindent 2 }} 104 | {{- end }} 105 | ipAddresses: 106 | - "127.0.0.1" 107 | {{- with .Values.etcd.service.loadBalancerIP }} 108 | {{- if not (has . $.Values.etcd.certSANs.ipAddresses) }} 109 | - {{ . | quote }} 110 | {{- end }} 111 | {{- end }} 112 | {{- with .Values.etcd.certSANs.ipAddresses }} 113 | {{- . | toYaml | nindent 2 }} 114 | {{- end }} 115 | issuerRef: 116 | name: "{{ $fullName }}-etcd-issuer" 117 | kind: Issuer 118 | --- 119 | apiVersion: cert-manager.io/v1 120 | kind: Certificate 121 | metadata: 122 | name: "{{ $fullName }}-pki-etcd-healthcheck-client" 123 | spec: 124 | commonName: "{{ $certName }}-etcd-healthcheck-client" 125 | secretName: "{{ $fullName }}-pki-etcd-healthcheck-client" 126 | duration: 8760h # 365d 127 | renewBefore: 4380h # 178d 128 | subject: 129 | organizations: 130 | - "system:masters" 131 | usages: 132 | - "signing" 133 | - "key encipherment" 134 | - "client auth" 135 | issuerRef: 136 | name: "{{ $fullName }}-etcd-issuer" 137 | kind: Issuer 138 | --- 139 | apiVersion: cert-manager.io/v1 140 | kind: Certificate 141 | metadata: 142 | name: "{{ $fullName }}-pki-apiserver-etcd-client" 143 | spec: 144 | commonName: "{{ $certName }}-apiserver-etcd-client" 145 | secretName: "{{ $fullName }}-pki-apiserver-etcd-client" 146 | duration: 8760h # 365d 147 | renewBefore: 4380h # 178d 148 | subject: 149 | organizations: 150 | - "system:masters" 151 | usages: 152 | - "signing" 153 | - "key encipherment" 154 | - "client auth" 155 | issuerRef: 156 | name: "{{ $fullName }}-etcd-issuer" 157 | kind: Issuer 158 | {{- end }} 159 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/scheduler-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.scheduler.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: "{{ $fullName }}-scheduler" 8 | labels: 9 | app: "{{ $fullName }}-scheduler" 10 | {{- with .Values.scheduler.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.scheduler.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | replicas: {{ .Values.scheduler.replicaCount }} 19 | selector: 20 | matchLabels: 21 | app: "{{ $fullName }}-scheduler" 22 | template: 23 | metadata: 24 | labels: 25 | app: "{{ $fullName }}-scheduler" 26 | {{- with .Values.scheduler.podLabels }} 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- with .Values.scheduler.podAnnotations }} 30 | annotations: 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with .Values.scheduler.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.scheduler.tolerations }} 39 | tolerations: 40 | {{- toYaml . | nindent 6 }} 41 | {{- end }} 42 | {{- if or .Values.scheduler.affinity .Values.scheduler.podAntiAffinity }} 43 | affinity: 44 | {{- with .Values.scheduler.affinity }} 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | {{- if eq .Values.scheduler.podAntiAffinity "hard" }} 48 | podAntiAffinity: 49 | requiredDuringSchedulingIgnoredDuringExecution: 50 | - topologyKey: "{{ .Values.scheduler.podAntiAffinityTopologyKey }}" 51 | labelSelector: 52 | matchLabels: 53 | app: {{ $fullName }}-scheduler 54 | {{- else if eq .Values.scheduler.podAntiAffinity "soft" }} 55 | podAntiAffinity: 56 | preferredDuringSchedulingIgnoredDuringExecution: 57 | - weight: 1 58 | podAffinityTerm: 59 | topologyKey: "{{ .Values.scheduler.podAntiAffinityTopologyKey }}" 60 | labelSelector: 61 | matchLabels: 62 | app: {{ $fullName }}-scheduler 63 | {{- end }} 64 | {{- end }} 65 | {{- with .Values.scheduler.image.pullSecrets }} 66 | imagePullSecrets: 67 | {{- toYaml . | nindent 10 }} 68 | {{- end }} 69 | automountServiceAccountToken: false 70 | containers: 71 | - command: 72 | - kube-scheduler 73 | - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf 74 | - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf 75 | - --bind-address=0.0.0.0 76 | - --kubeconfig=/etc/kubernetes/scheduler.conf 77 | - --leader-elect=true 78 | - --secure-port={{ .Values.scheduler.port }} 79 | - --tls-cert-file=/pki/scheduler-server/tls.crt 80 | - --tls-private-key-file=/pki/scheduler-server/tls.key 81 | {{- range $key, $value := .Values.scheduler.extraArgs }} 82 | - --{{ $key }}={{ $value }} 83 | {{- end }} 84 | {{- with .Values.scheduler.image }} 85 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 86 | imagePullPolicy: {{ .pullPolicy }} 87 | {{- end }} 88 | livenessProbe: 89 | failureThreshold: 8 90 | httpGet: 91 | path: /healthz 92 | port: {{ .Values.scheduler.port }} 93 | scheme: HTTPS 94 | initialDelaySeconds: 15 95 | timeoutSeconds: 15 96 | name: kube-scheduler 97 | resources: 98 | {{- toYaml .Values.scheduler.resources | nindent 10 }} 99 | {{- with .Values.scheduler.extraEnv }} 100 | env: 101 | {{- toYaml . | nindent 8 }} 102 | {{- end }} 103 | volumeMounts: 104 | - mountPath: /etc/kubernetes/ 105 | name: kubeconfig 106 | readOnly: true 107 | - mountPath: /pki/scheduler-server 108 | name: pki-scheduler-server 109 | - mountPath: /pki/scheduler-client 110 | name: pki-scheduler-client 111 | {{- with .Values.scheduler.extraVolumeMounts }} 112 | {{- toYaml . | nindent 8 }} 113 | {{- end }} 114 | {{- with .Values.scheduler.sidecars }} 115 | {{- toYaml . | nindent 6 }} 116 | {{- end }} 117 | securityContext: 118 | seccompProfile: 119 | type: RuntimeDefault 120 | volumes: 121 | - configMap: 122 | name: "{{ $fullName }}-scheduler-conf" 123 | name: kubeconfig 124 | - secret: 125 | secretName: "{{ $fullName }}-pki-scheduler-server" 126 | name: pki-scheduler-server 127 | - secret: 128 | secretName: "{{ $fullName }}-pki-scheduler-client" 129 | name: pki-scheduler-client 130 | 131 | {{- with .Values.scheduler.extraVolumes }} 132 | {{- toYaml . | nindent 6 }} 133 | {{- end }} 134 | {{- end }} 135 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/admin-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.admin.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: "{{ $fullName }}-admin" 8 | labels: 9 | app: "{{ $fullName }}-admin" 10 | {{- with .Values.admin.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.admin.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | replicas: {{ .Values.admin.replicaCount }} 19 | selector: 20 | matchLabels: 21 | app: "{{ $fullName }}-admin" 22 | template: 23 | metadata: 24 | labels: 25 | app: "{{ $fullName }}-admin" 26 | {{- with .Values.admin.podLabels }} 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- with .Values.admin.podAnnotations }} 30 | annotations: 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with .Values.admin.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.admin.tolerations }} 39 | tolerations: 40 | {{- toYaml . | nindent 6 }} 41 | {{- end }} 42 | {{- if or .Values.admin.affinity .Values.admin.podAntiAffinity }} 43 | affinity: 44 | {{- with .Values.admin.affinity }} 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | {{- if eq .Values.admin.podAntiAffinity "hard" }} 48 | podAntiAffinity: 49 | requiredDuringSchedulingIgnoredDuringExecution: 50 | - topologyKey: "{{ .Values.admin.podAntiAffinityTopologyKey }}" 51 | labelSelector: 52 | matchLabels: 53 | app: {{ $fullName }}-admin 54 | {{- else if eq .Values.admin.podAntiAffinity "soft" }} 55 | podAntiAffinity: 56 | preferredDuringSchedulingIgnoredDuringExecution: 57 | - weight: 1 58 | podAffinityTerm: 59 | topologyKey: "{{ .Values.admin.podAntiAffinityTopologyKey }}" 60 | labelSelector: 61 | matchLabels: 62 | app: {{ $fullName }}-admin 63 | {{- end }} 64 | {{- end }} 65 | {{- with .Values.admin.image.pullSecrets }} 66 | imagePullSecrets: 67 | {{- toYaml . | nindent 10 }} 68 | {{- end }} 69 | automountServiceAccountToken: false 70 | terminationGracePeriodSeconds: 5 71 | containers: 72 | - command: [ 'sleep', 'infinity' ] 73 | {{- with .Values.admin.image }} 74 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 75 | imagePullPolicy: {{ .pullPolicy }} 76 | {{- end }} 77 | name: admin 78 | readinessProbe: 79 | exec: 80 | command: 81 | - kubectl 82 | - auth 83 | - can-i 84 | - '*' 85 | - '*' 86 | initialDelaySeconds: 15 87 | periodSeconds: 5 88 | resources: 89 | {{- toYaml .Values.admin.resources | nindent 10 }} 90 | env: 91 | - name: KUBECONFIG 92 | value: "/etc/kubernetes/admin.conf" 93 | {{- with .Values.admin.extraEnv }} 94 | {{- toYaml . | nindent 8 }} 95 | {{- end }} 96 | volumeMounts: 97 | - mountPath: /etc/kubernetes/ 98 | name: kubeconfig 99 | readOnly: true 100 | - mountPath: /pki/admin-client 101 | name: pki-admin-client 102 | - mountPath: /scripts 103 | name: scripts 104 | {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 105 | - mountPath: /manifests 106 | name: manifests 107 | {{- end }} 108 | - mountPath: /config 109 | name: config 110 | {{- with .Values.admin.extraVolumeMounts }} 111 | {{- toYaml . | nindent 8 }} 112 | {{- end }} 113 | {{- with .Values.admin.sidecars }} 114 | {{- toYaml . | nindent 6 }} 115 | {{- end }} 116 | securityContext: 117 | seccompProfile: 118 | type: RuntimeDefault 119 | volumes: 120 | - configMap: 121 | name: "{{ $fullName }}-admin-conf" 122 | name: kubeconfig 123 | - secret: 124 | secretName: "{{ $fullName }}-pki-admin-client" 125 | name: pki-admin-client 126 | - name: scripts 127 | configMap: 128 | name: "{{ $fullName }}-kubeadm-scripts" 129 | defaultMode: 0777 130 | {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 131 | - name: manifests 132 | projected: 133 | sources: 134 | {{- if or .Values.extraManifests }} 135 | - secret: 136 | name: "{{ $fullName }}-extra-manifests" 137 | {{- end }} 138 | {{- if or .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }} 139 | - configMap: 140 | name: "{{ $fullName }}-konnectivity-manifests" 141 | {{- end }} 142 | {{- end }} 143 | - name: config 144 | configMap: 145 | name: "{{ $fullName }}-kubeadm-config" 146 | {{- with .Values.admin.extraVolumes }} 147 | {{- toYaml . | nindent 6 }} 148 | {{- end }} 149 | {{- end }} 150 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/manifests/coredns.yaml: -------------------------------------------------------------------------------- 1 | # Source: https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/coredns/coredns.yaml.base 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: coredns 7 | namespace: kube-system 8 | labels: 9 | kubernetes.io/cluster-service: "true" 10 | addonmanager.kubernetes.io/mode: Reconcile 11 | --- 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: ClusterRole 14 | metadata: 15 | labels: 16 | kubernetes.io/bootstrapping: rbac-defaults 17 | addonmanager.kubernetes.io/mode: Reconcile 18 | name: system:coredns 19 | rules: 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - endpoints 24 | - services 25 | - pods 26 | - namespaces 27 | verbs: 28 | - list 29 | - watch 30 | - apiGroups: 31 | - "" 32 | resources: 33 | - nodes 34 | verbs: 35 | - get 36 | - apiGroups: 37 | - discovery.k8s.io 38 | resources: 39 | - endpointslices 40 | verbs: 41 | - list 42 | - watch 43 | --- 44 | apiVersion: rbac.authorization.k8s.io/v1 45 | kind: ClusterRoleBinding 46 | metadata: 47 | annotations: 48 | rbac.authorization.kubernetes.io/autoupdate: "true" 49 | labels: 50 | kubernetes.io/bootstrapping: rbac-defaults 51 | addonmanager.kubernetes.io/mode: EnsureExists 52 | name: system:coredns 53 | roleRef: 54 | apiGroup: rbac.authorization.k8s.io 55 | kind: ClusterRole 56 | name: system:coredns 57 | subjects: 58 | - kind: ServiceAccount 59 | name: coredns 60 | namespace: kube-system 61 | --- 62 | apiVersion: v1 63 | kind: ConfigMap 64 | metadata: 65 | name: coredns 66 | namespace: kube-system 67 | data: 68 | Corefile: | 69 | .:53 { 70 | errors 71 | health { 72 | lameduck 5s 73 | } 74 | ready 75 | kubernetes {{ .Values.networking.dnsDomain }} in-addr.arpa ip6.arpa { 76 | pods insecure 77 | fallthrough in-addr.arpa ip6.arpa 78 | ttl 30 79 | } 80 | prometheus :9153 81 | forward . /etc/resolv.conf { 82 | max_concurrent 1000 83 | } 84 | cache 30 85 | loop 86 | reload 87 | loadbalance 88 | } 89 | --- 90 | apiVersion: apps/v1 91 | kind: Deployment 92 | metadata: 93 | name: coredns 94 | namespace: kube-system 95 | labels: 96 | k8s-app: kube-dns 97 | kubernetes.io/cluster-service: "true" 98 | addonmanager.kubernetes.io/mode: Reconcile 99 | kubernetes.io/name: "CoreDNS" 100 | spec: 101 | replicas: {{ .Values.coredns.replicaCount }} 102 | strategy: 103 | type: RollingUpdate 104 | rollingUpdate: 105 | maxUnavailable: 1 106 | selector: 107 | matchLabels: 108 | k8s-app: kube-dns 109 | template: 110 | metadata: 111 | labels: 112 | k8s-app: kube-dns 113 | spec: 114 | securityContext: 115 | seccompProfile: 116 | type: RuntimeDefault 117 | priorityClassName: system-cluster-critical 118 | serviceAccountName: coredns 119 | affinity: 120 | podAntiAffinity: 121 | preferredDuringSchedulingIgnoredDuringExecution: 122 | - weight: 100 123 | podAffinityTerm: 124 | labelSelector: 125 | matchExpressions: 126 | - key: k8s-app 127 | operator: In 128 | values: ["kube-dns"] 129 | topologyKey: kubernetes.io/hostname 130 | tolerations: 131 | - key: "CriticalAddonsOnly" 132 | operator: "Exists" 133 | nodeSelector: 134 | kubernetes.io/os: linux 135 | {{- with .Values.coredns.image.pullSecrets }} 136 | imagePullSecrets: 137 | {{- toYaml . | nindent 10 }} 138 | {{- end }} 139 | containers: 140 | - name: coredns 141 | {{- with .Values.coredns.image }} 142 | image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}" 143 | imagePullPolicy: {{ .pullPolicy }} 144 | {{- end }} 145 | resources: 146 | {{- toYaml .Values.coredns.resources | nindent 10 }} 147 | args: [ "-conf", "/etc/coredns/Corefile" ] 148 | volumeMounts: 149 | - name: config-volume 150 | mountPath: /etc/coredns 151 | readOnly: true 152 | ports: 153 | - containerPort: 53 154 | name: dns 155 | protocol: UDP 156 | - containerPort: 53 157 | name: dns-tcp 158 | protocol: TCP 159 | - containerPort: 9153 160 | name: metrics 161 | protocol: TCP 162 | livenessProbe: 163 | httpGet: 164 | path: /health 165 | port: 8080 166 | scheme: HTTP 167 | initialDelaySeconds: 60 168 | timeoutSeconds: 5 169 | successThreshold: 1 170 | failureThreshold: 5 171 | readinessProbe: 172 | httpGet: 173 | path: /ready 174 | port: 8181 175 | scheme: HTTP 176 | securityContext: 177 | allowPrivilegeEscalation: false 178 | capabilities: 179 | add: 180 | - NET_BIND_SERVICE 181 | drop: 182 | - all 183 | readOnlyRootFilesystem: true 184 | dnsPolicy: Default 185 | volumes: 186 | - name: config-volume 187 | configMap: 188 | name: coredns 189 | items: 190 | - key: Corefile 191 | path: Corefile 192 | --- 193 | apiVersion: v1 194 | kind: Service 195 | metadata: 196 | name: kube-dns 197 | namespace: kube-system 198 | annotations: 199 | prometheus.io/port: "9153" 200 | prometheus.io/scrape: "true" 201 | labels: 202 | k8s-app: kube-dns 203 | kubernetes.io/cluster-service: "true" 204 | addonmanager.kubernetes.io/mode: Reconcile 205 | kubernetes.io/name: "CoreDNS" 206 | spec: 207 | selector: 208 | k8s-app: kube-dns 209 | clusterIP: {{ template "kubernetes.getCoreDNS" . }} 210 | ports: 211 | - name: dns 212 | port: 53 213 | protocol: UDP 214 | - name: dns-tcp 215 | port: 53 216 | protocol: TCP 217 | - name: metrics 218 | port: 9153 219 | protocol: TCP 220 | -------------------------------------------------------------------------------- /deploy/helm/kubernetes/templates/etcd-backup-cronjob.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.etcd.backup.enabled }} 2 | {{- $fullName := include "kubernetes.fullname" . -}} 3 | --- 4 | apiVersion: batch/v1 5 | kind: CronJob 6 | metadata: 7 | name: {{ $fullName }}-etcd-backup 8 | labels: 9 | app: {{ $fullName }}-etcd-backup 10 | {{- with .Values.etcd.backup.labels }} 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.etcd.backup.annotations }} 14 | annotations: 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | schedule: "{{ .Values.etcd.backup.schedule }}" 19 | successfulJobsHistoryLimit: {{ .Values.etcd.backup.successfulJobsHistoryLimit }} 20 | failedJobsHistoryLimit: {{ .Values.etcd.backup.failedJobsHistoryLimit }} 21 | jobTemplate: 22 | metadata: 23 | labels: 24 | app: {{ $fullName }}-etcd-backup 25 | {{- with .Values.etcd.backup.labels }} 26 | {{- toYaml . | nindent 8 }} 27 | {{- end }} 28 | {{- with .Values.etcd.backup.annotations }} 29 | annotations: 30 | {{- toYaml . | nindent 8 }} 31 | {{- end }} 32 | spec: 33 | template: 34 | metadata: 35 | labels: 36 | app: {{ $fullName }}-etcd-backup 37 | {{- with .Values.etcd.backup.podLabels }} 38 | {{- toYaml . | nindent 12 }} 39 | {{- end }} 40 | {{- with .Values.etcd.backup.podAnnotations }} 41 | annotations: 42 | {{- toYaml . | nindent 12 }} 43 | {{- end }} 44 | spec: 45 | {{- with .Values.etcd.backup.nodeSelector }} 46 | nodeSelector: 47 | {{- toYaml . | nindent 12 }} 48 | {{- end }} 49 | {{- with .Values.etcd.backup.tolerations }} 50 | tolerations: 51 | {{- toYaml . | nindent 10 }} 52 | {{- end }} 53 | {{- if or .Values.etcd.backup.affinity .Values.etcd.backup.podAffinity }} 54 | affinity: 55 | {{- with .Values.etcd.backup.affinity }} 56 | {{- toYaml . | nindent 12 }} 57 | {{- end }} 58 | {{- if eq .Values.etcd.backup.podAffinity "hard" }} 59 | podAffinity: 60 | requiredDuringSchedulingIgnoredDuringExecution: 61 | - topologyKey: "{{ .Values.etcd.backup.podAffinityTopologyKey }}" 62 | labelSelector: 63 | matchLabels: 64 | app: {{ $fullName }}-etcd 65 | {{- else if eq .Values.etcd.backup.podAffinity "soft" }} 66 | podAffinity: 67 | preferredDuringSchedulingIgnoredDuringExecution: 68 | - weight: 1 69 | podAffinityTerm: 70 | topologyKey: "{{ .Values.etcd.backup.podAffinityTopologyKey }}" 71 | labelSelector: 72 | matchLabels: 73 | app: {{ $fullName }}-etcd 74 | {{- end }} 75 | {{- end }} 76 | 77 | {{- with .Values.etcd.image.pullSecrets }} 78 | imagePullSecrets: 79 | {{- toYaml . | nindent 10 }} 80 | {{- end }} 81 | automountServiceAccountToken: false 82 | restartPolicy: OnFailure 83 | containers: 84 | - command: 85 | - /bin/sh 86 | - -xc 87 | - | 88 | rtc() { while read k s v; do test "$k" = "rtc_$1" && echo "$v" && break; done