├── .github └── workflows │ └── go.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── charts ├── contour │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── crds │ │ ├── extensionservices.yaml │ │ ├── httpproxies.yaml │ │ └── tlscertificatedeligations.yaml │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── configmap.yaml │ │ ├── contour-deploy.yaml │ │ ├── envoy-deploy.yaml │ │ ├── envoy-ds.yaml │ │ ├── job.yaml │ │ ├── rbac.yaml │ │ ├── service.yaml │ │ └── servicemonitor.yaml │ └── values.yaml ├── kok-operator │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── deployment.yaml │ │ ├── rbac.yaml │ │ └── service.yaml │ └── values.yaml └── local-path-provisioner │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── clusterrole.yaml │ ├── clusterrolebinding.yaml │ ├── configmap.yaml │ ├── deployment.yaml │ ├── registry-secret.yaml │ ├── serviceaccount.yaml │ └── storageclass.yaml │ └── values.yaml ├── cmd └── controller │ ├── app │ ├── app_option │ │ └── option.go │ ├── certs.go │ ├── controller.go │ ├── fake_apiserver.go │ ├── root.go │ └── version.go │ └── main.go ├── config ├── crd │ └── bases │ │ ├── devops.fake.io_clustercredentials.yaml │ │ ├── devops.fake.io_clusters.yaml │ │ ├── devops.fake.io_machines.yaml │ │ └── workload.fake.io_addons.yaml └── rbac │ └── role.yaml ├── doc ├── addons.md ├── images │ ├── node.jpeg │ └── pods.jpeg ├── k3s.md └── module.md ├── docker ├── base │ └── Dockerfile ├── coredns │ └── Dockerfile ├── kok-base │ └── Dockerfile ├── kok-operator │ └── Dockerfile ├── kubernetes │ └── Dockerfile └── metrics-server │ └── Dockerfile ├── go.mod ├── go.sum ├── hack └── boilerplate.go.txt ├── manifests ├── addons │ ├── cilium-v1.9.yaml │ ├── contour-v1.11.0.yaml │ ├── contour.yaml │ ├── coredns.yaml │ ├── kube-flannel.yaml │ ├── kube-vip-243.yaml │ ├── kube-vip.yaml │ ├── local-storage.yaml │ └── metrics-server.yaml ├── etcd-statefulset.yaml ├── ha-local-cluster-node.yaml ├── ha-local-cluster.yaml ├── managed-cluster-node.yaml ├── managed-cluster.yaml ├── test │ ├── cilium-connectivity-check.yaml │ ├── curl-deployment.yaml │ ├── etcd-statefulset-pv.yaml │ ├── kuard.yaml │ └── local-cluster.yaml └── vcluster │ ├── readme.md │ └── vcluster.yaml ├── pkg ├── addons │ ├── coredns │ │ └── coredns.go │ ├── etcd │ │ └── etcd.go │ ├── flannel │ │ └── flannel.go │ ├── kubeproxy │ │ └── kubeproxy.go │ ├── kubevip │ │ └── kubevip.go │ ├── metallb │ │ └── doc.go │ ├── metricsserver │ │ └── metricsserver.go │ └── rawcni │ │ └── rawcni.go ├── apis │ ├── devops │ │ └── v1 │ │ │ ├── clusterCredential_types.go │ │ │ ├── cluster_types.go │ │ │ ├── devops_helper.go │ │ │ ├── groupversion_info.go │ │ │ ├── machine_types.go │ │ │ └── zz_generated.deepcopy.go │ ├── helper.go │ ├── types.go │ └── workload │ │ └── v1 │ │ ├── addons_types.go │ │ ├── groupversion_info.go │ │ └── zz_generated.deepcopy.go ├── apiserver │ ├── apiserver.go │ └── internal │ │ ├── apiserver.go │ │ ├── control_plane.go │ │ ├── etcd.go │ │ ├── process.go │ │ └── tinyca.go ├── clustermanager │ ├── cluster.go │ ├── interface.go │ └── manager.go ├── constants │ ├── constants.go │ ├── finalizers.go │ ├── labels.go │ ├── misc.go │ ├── ports.go │ └── version.go ├── controllers │ ├── addons │ │ └── addons.go │ ├── cluster │ │ ├── cluster_controller.go │ │ └── cluster_helper.go │ ├── common │ │ ├── defaults.go │ │ ├── helper.go │ │ └── templates.go │ ├── controller.go │ └── machine │ │ ├── machine_controller.go │ │ └── machine_helper.go ├── gmanager │ └── gmanager.go ├── helm │ ├── object │ │ ├── objects.go │ │ └── render.go │ └── v3 │ │ ├── action.go │ │ ├── client.go │ │ └── repo.go ├── k8sclient │ ├── client.go │ └── scheme.go ├── k8sutil │ ├── crds.go │ ├── objects.go │ ├── resource.go │ ├── util.go │ └── util_test.go ├── option │ ├── apiserver.go │ ├── controller.go │ └── global.go ├── provider │ ├── baremetal │ │ ├── baremetal.go │ │ ├── cluster │ │ │ ├── create.go │ │ │ ├── handler.go │ │ │ ├── provider.go │ │ │ └── update.go │ │ ├── machine │ │ │ ├── handler.go │ │ │ └── provider.go │ │ └── validation │ │ │ ├── cluster.go │ │ │ └── machine.go │ ├── cluster │ │ ├── cluster.go │ │ └── interface.go │ ├── config │ │ └── config.go │ ├── machine │ │ ├── interface.go │ │ └── machine.go │ ├── managed │ │ ├── cluster │ │ │ ├── handler.go │ │ │ ├── helper.go │ │ │ └── provider.go │ │ ├── machine │ │ │ ├── handler.go │ │ │ └── provider.go │ │ └── managed.go │ ├── phases │ │ ├── certs │ │ │ ├── certlist.go │ │ │ ├── certs.go │ │ │ └── kubeconfig.go │ │ ├── clean │ │ │ └── clean.go │ │ ├── cri │ │ │ └── containerd.go │ │ ├── gpu │ │ │ └── gpu.go │ │ ├── join │ │ │ ├── helper.go │ │ │ └── join_node.go │ │ ├── kubeadm │ │ │ ├── action.go │ │ │ └── config.go │ │ ├── kubebin │ │ │ └── kubebin.go │ │ ├── kubemisc │ │ │ ├── helpers.go │ │ │ └── kubemisc.go │ │ └── system │ │ │ ├── centos.go │ │ │ ├── debian.go │ │ │ ├── system.go │ │ │ └── ubuntu.go │ ├── preflight │ │ └── checks.go │ └── provider.go ├── static │ ├── crds.go │ └── crds │ │ ├── devops.fake.io_clustercredentials.yaml │ │ ├── devops.fake.io_clusters.yaml │ │ ├── devops.fake.io_machines.yaml │ │ └── workload.fake.io_addons.yaml ├── util │ ├── allocator │ │ ├── bitmap.go │ │ ├── interfaces.go │ │ └── utils.go │ ├── apiclient │ │ ├── client.go │ │ ├── idempotency.go │ │ ├── install.go │ │ └── version.go │ ├── hash │ │ └── hash.go │ ├── hosts │ │ ├── hosts.go │ │ ├── local.go │ │ └── remote.go │ ├── ipallocator │ │ └── allocator.go │ ├── kubeconfig │ │ └── kubeconfig.go │ ├── pkiutil │ │ ├── constants.go │ │ ├── endpoint.go │ │ └── pki_helpers.go │ ├── pointer │ │ └── pointer.go │ ├── ssh │ │ ├── helpers.go │ │ └── ssh.go │ ├── supervisor │ │ └── systemd.go │ ├── template │ │ └── template.go │ └── validation │ │ ├── kubernetes.go │ │ ├── name.go │ │ ├── url.go │ │ └── validation.go └── version │ └── version.go └── tools ├── image.sh ├── init.sh ├── k3s-install.sh ├── k3s.sh ├── kubeadm-config.yaml ├── kubeadmin-config.yaml └── new-init.sh /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: Build 34 | run: make build-local 35 | 36 | # - name: Test 37 | # run: go test -v ./... 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | .idea/ 16 | .vscode/ 17 | vendor/ 18 | bin/ 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.13 as builder 3 | 4 | WORKDIR /workspace 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | # cache deps before building and copying source so that we don't need to re-download as much 9 | # and so that source changes don't invalidate our downloaded layer 10 | RUN go mod download 11 | 12 | # Copy the go source 13 | COPY main.go main.go 14 | COPY api/ api/ 15 | COPY controllers/ controllers/ 16 | 17 | # Build 18 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go 19 | 20 | # Use distroless as minimal base image to package the manager binary 21 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 22 | FROM gcr.io/distroless/static:nonroot 23 | WORKDIR / 24 | COPY --from=builder /workspace/manager . 25 | USER nonroot:nonroot 26 | 27 | ENTRYPOINT ["/manager"] 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION ?= v0.2.0 2 | # Image URL to use all building/pushing image targets 3 | IMG_REG ?= docker.io/wtxue 4 | IMG_CTL := $(IMG_REG)/kok-operator 5 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 6 | CRD_OPTIONS ?= "crd:trivialVersions=true" 7 | 8 | # This repo's root import path (under GOPATH). 9 | ROOT := github.com/wtxue/kok-operator 10 | 11 | GO_VERSION := 1.19.0 12 | ARCH ?= $(shell go env GOARCH) 13 | BUILD_DATE = $(shell date +'%Y-%m-%dT%H:%M:%SZ') 14 | COMMIT = $(shell git rev-parse --short HEAD) 15 | GOENV := CGO_ENABLED=0 GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) GOPROXY=https://goproxy.io,direct 16 | GO := $(GOENV) go build -tags=jsoniter 17 | 18 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 19 | ifeq (,$(shell go env GOBIN)) 20 | GOBIN=$(shell go env GOPATH)/bin 21 | else 22 | GOBIN=$(shell go env GOBIN) 23 | endif 24 | 25 | all: build 26 | 27 | # Run tests 28 | test: generate fmt vet manifests 29 | go test ./... -coverprofile cover.out 30 | 31 | # Run against the configured Kubernetes cluster in ~/.kube/config 32 | run: generate fmt vet manifests 33 | go run ./main.go 34 | 35 | # generate crd spec and deepcopy 36 | crd: generate manifests 37 | 38 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 39 | deploy: manifests 40 | # cd config/manager && kustomize edit set image controller=${IMG} 41 | # kustomize build config/default | kubectl apply -f - 42 | 43 | # Generate manifests e.g. CRD, RBAC etc. 44 | manifests: controller-gen 45 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases 46 | 47 | # Run go fmt against code 48 | fmt: 49 | go fmt ./... 50 | 51 | # Run go vet against code 52 | vet: 53 | go vet ./... 54 | 55 | # Generate code 56 | generate: controller-gen 57 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 58 | 59 | # Build the docker image 60 | docker-build: 61 | docker run --rm -v "$$PWD":/go/src/${ROOT} -v ${GOPATH}/pkg/mod:/go/pkg/mod -w /go/src/${ROOT} golang:${GO_VERSION} make build-local 62 | 63 | build: build-local 64 | 65 | build-local: 66 | $(GO) -v -o bin/kok-operator -ldflags "-s -w -X $(ROOT)/pkg/version.Release=$(VERSION) -X $(ROOT)/pkg/version.Commit=$(COMMIT) \ 67 | -X $(ROOT)/pkg/version.BuildDate=$(BUILD_DATE)" cmd/controller/main.go 68 | 69 | build-linux: 70 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ 71 | go build -tags=jsoniter -v -o bin/kok-operator -ldflags "-s -w -X $(ROOT)/pkg/version.Release=$(VERSION) -X $(ROOT)/pkg/version.Commit=$(COMMIT) \ 72 | -X $(ROOT)/pkg/version.BuildDate=$(BUILD_DATE)" cmd/controller/main.go 73 | 74 | 75 | # Push the docker image 76 | push: 77 | docker build -t ${IMG_CTL}:${VERSION} -f ./docker/kok-operator/Dockerfile . 78 | docker push ${IMG_CTL}:${VERSION} 79 | 80 | # find or download controller-gen 81 | # download controller-gen if necessary 82 | controller-gen: 83 | ifeq (, $(shell which controller-gen)) 84 | @{ \ 85 | set -e ;\ 86 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ 87 | cd $$CONTROLLER_GEN_TMP_DIR ;\ 88 | go mod init tmp ;\ 89 | go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2 ;\ 90 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ 91 | } 92 | CONTROLLER_GEN=$(GOBIN)/controller-gen 93 | else 94 | CONTROLLER_GEN=$(shell which controller-gen) 95 | endif 96 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: k8s.io 2 | repo: github.com/wtxue/kok-operator 3 | resources: 4 | - group: devops 5 | kind: Cluster 6 | version: v1 7 | - group: devops 8 | kind: Machine 9 | version: v1 10 | version: "2" 11 | -------------------------------------------------------------------------------- /charts/contour/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /charts/contour/Chart.yaml: -------------------------------------------------------------------------------- 1 | annotations: 2 | category: Infrastructure 3 | apiVersion: v1 4 | appVersion: 1.6.1 5 | description: Contour Ingress controller for Kubernetes 6 | home: https://projectcontour.io 7 | icon: https://bitnami.com/assets/stacks/contour/img/contour-stack-220x234.png 8 | keywords: 9 | - ingress 10 | - envoy 11 | - contour 12 | name: contour 13 | sources: 14 | - https://github.com/projectcontour/contour 15 | - https://github.com/envoyproxy/envoy 16 | version: 1.1.5 17 | -------------------------------------------------------------------------------- /charts/contour/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | {{- if eq .Values.envoy.service.type "LoadBalancer" }} 2 | 1. Get Contours's load balancer IP/hostname: 3 | 4 | NOTE: It may take a few minutes for this to become available. 5 | 6 | You can watch the status by running: 7 | 8 | $ kubectl get svc {{ include "contour.fullname" . }} --namespace {{ .Release.Namespace }} -w 9 | 10 | Once 'EXTERNAL-IP' is no longer '': 11 | 12 | $ kubectl describe svc {{ include "contour.fullname" . }} --namespace {{ .Release.Namespace }} | grep Ingress | awk '{print $3}' 13 | 14 | 2. Configure DNS records corresponding to Kubernetes ingress resources to point to the load balancer IP/hostname found in step 1 15 | {{- end }} 16 | {{- if eq .Values.envoy.service.type "NodePort" }} 17 | {{- if (and (not (empty .Values.envoy.service.nodePorts.https)) (not (empty .Values.envoy.service.nodePorts.http)))}} 18 | 1. Contour is listening on the following ports on the host machine: 19 | 20 | http - {{ .Values.envoy.service.nodePorts.http }} 21 | https - {{ .Values.envoy.service.nodePorts.https }} 22 | {{- else }} 23 | 1. Contour has been started. You can find out the port numbers being used by Contour by running: 24 | 25 | $ kubectl describe svc {{ include "contour.fullname" . }} --namespace {{ .Release.Namespace }} 26 | 27 | {{- end }} 28 | 29 | 2. Configure DNS records corresponding to Kubernetes ingress resources to point to the NODE_IP/NODE_HOST 30 | {{- end }} -------------------------------------------------------------------------------- /charts/contour/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.configInline }} 2 | --- 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: {{ include "contour.fullname" . }} 7 | labels: {{- include "contour.labels" . | nindent 4}} 8 | app.kubernetes.io/component: contour 9 | data: 10 | contour.yaml: | 11 | {{ include "contour.tplValue" ( dict "value" .Values.configInline "context" $) | indent 4 }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /charts/contour/templates/job.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.contour.enabled }} 2 | --- 3 | apiVersion: batch/v1 4 | kind: Job 5 | metadata: 6 | name: {{ include "contour.fullname" . }}-contour-certgen 7 | labels: {{- include "contour.labels" . | nindent 4 }} 8 | app.kubernetes.io/component: contour-certgen 9 | spec: 10 | ttlSecondsAfterFinished: 0 11 | template: 12 | metadata: 13 | labels: {{- include "contour.labels" . | nindent 8 }} 14 | app.kubernetes.io/component: contour-certgen 15 | spec: 16 | {{- include "contour.imagePullSecrets" . | nindent 6 }} 17 | {{- with .Values.contour.affinity }} 18 | affinity: 19 | {{ toYaml . | indent 8 }} 20 | {{- end }} 21 | {{- with .Values.contour.nodeSelector }} 22 | nodeSelector: 23 | {{ toYaml . | indent 8 }} 24 | {{- end }} 25 | {{- with .Values.contour.tolerations }} 26 | tolerations: 27 | {{ toYaml . | indent 8 }} 28 | {{- end }} 29 | containers: 30 | - name: contour 31 | image: {{ include "contour.image" . }} 32 | imagePullPolicy: {{ .Values.contour.image.pullPolicy }} 33 | command: 34 | - contour 35 | args: 36 | - certgen 37 | - --kube 38 | - --incluster 39 | - --overwrite 40 | - --secrets-format=compact 41 | - --namespace=$(CONTOUR_NAMESPACE) 42 | env: 43 | - name: CONTOUR_NAMESPACE 44 | valueFrom: 45 | fieldRef: 46 | fieldPath: metadata.namespace 47 | resources: 48 | {{ toYaml .Values.contour.resources | indent 10 }} 49 | restartPolicy: Never 50 | serviceAccountName: {{ include "contour.contourCertGenServiceAccountName" . }} 51 | {{- if .Values.contour.securityContext.enabled }} 52 | securityContext: 53 | runAsUser: {{ .Values.contour.securityContext.runAsUser }} 54 | runAsGroup: {{ .Values.contour.securityContext.runAsGroup }} 55 | fsGroup: {{ .Values.contour.securityContext.fsGroup }} 56 | runAsNonRoot: {{ .Values.contour.securityContext.runAsNonRoot }} 57 | {{- end }} 58 | parallelism: 1 59 | completions: 1 60 | backoffLimit: 1 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /charts/contour/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.contour.enabled }} 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: {{ include "contour.fullname" . }} 7 | labels: {{- include "contour.labels" . | nindent 4 }} 8 | app.kubernetes.io/component: contour 9 | spec: 10 | ports: 11 | - port: 8001 12 | name: xds 13 | protocol: TCP 14 | targetPort: 8001 15 | - name: metrics 16 | port: 8000 17 | protocol: TCP 18 | targetPort: 8000 19 | selector: {{- include "contour.matchLabels" . | nindent 4 }} 20 | app.kubernetes.io/component: contour 21 | type: ClusterIP 22 | {{- end }} 23 | {{- if .Values.envoy.enabled }} 24 | --- 25 | apiVersion: v1 26 | kind: Service 27 | metadata: 28 | name: {{ include "contour.fullname" . }}-envoy 29 | labels: {{- include "contour.labels" . | nindent 4 }} 30 | app.kubernetes.io/component: envoy 31 | {{- if .Values.envoy.service.labels }} 32 | {{- include "contour.tplValue" (dict "value" .Values.envoy.service.labels "context" $) | nindent 4 }} 33 | {{- end }} 34 | annotations: 35 | {{- if .Values.envoy.service.annotations }} 36 | {{- include "contour.tplValue" (dict "value" .Values.envoy.service.annotations "context" $) | nindent 4 }} 37 | {{- end }} 38 | spec: 39 | {{- if .Values.envoy.service.externalTrafficPolicy }} 40 | externalTrafficPolicy: {{ .Values.envoy.service.externalTrafficPolicy | quote }} 41 | {{- end }} 42 | {{- if not (empty .Values.envoy.service.clusterIP) }} 43 | clusterIP: {{ .Values.envoy.service.clusterIP | quote }} 44 | {{- end }} 45 | {{- if .Values.envoy.service.externalIPs }} 46 | externalIPs: {{- toYaml .Values.envoy.service.externalIPs | nindent 4 }} 47 | {{- end }} 48 | {{- if .Values.envoy.service.loadBalancerIP }} 49 | loadBalancerIP: {{ .Values.envoy.service.loadBalancerIP | quote }} 50 | {{- end }} 51 | {{- if .Values.envoy.service.loadBalancerSourceRanges }} 52 | loadBalancerSourceRanges: {{- toYaml .Values.envoy.service.loadBalancerSourceRanges | nindent 4 }} 53 | {{- end }} 54 | ports: 55 | - name: http 56 | port: {{ .Values.envoy.service.ports.http }} 57 | protocol: TCP 58 | targetPort: http 59 | {{- if and (or (eq .Values.envoy.service.type "NodePort") (eq .Values.envoy.service.type "LoadBalancer")) (not (empty .Values.envoy.service.nodePorts.http)) }} 60 | nodePort: {{ .Values.envoy.service.nodePorts.http }} 61 | {{- else if eq .Values.envoy.service.type "ClusterIP" }} 62 | nodePort: null 63 | {{- end }} 64 | - name: https 65 | port: {{ .Values.envoy.service.ports.https }} 66 | protocol: TCP 67 | targetPort: https 68 | {{- if and (or (eq .Values.envoy.service.type "NodePort") (eq .Values.envoy.service.type "LoadBalancer")) (not (empty .Values.envoy.service.nodePorts.https)) }} 69 | nodePort: {{ .Values.envoy.service.nodePorts.https }} 70 | {{- else if eq .Values.envoy.service.type "ClusterIP" }} 71 | nodePort: null 72 | {{- end }} 73 | - name: metrics 74 | port: 8002 75 | protocol: TCP 76 | targetPort: 8002 77 | selector: {{- include "contour.matchLabels" . | nindent 4 }} 78 | app.kubernetes.io/component: envoy 79 | type: {{ .Values.envoy.service.type }} 80 | {{- end }} 81 | -------------------------------------------------------------------------------- /charts/contour/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.prometheus.serviceMonitor.enabled }} 2 | {{- if .Values.contour.enabled }} 3 | --- 4 | apiVersion: monitoring.coreos.com/v1 5 | kind: ServiceMonitor 6 | metadata: 7 | name: {{ include "contour.fullname" . }}-contour 8 | labels: {{- include "contour.labels" . | nindent 4 }} 9 | app.kubernetes.io/component: contour 10 | spec: 11 | jobLabel: {{ .Values.prometheus.serviceMonitor.jobLabel | quote }} 12 | selector: 13 | matchLabels: {{- include "contour.matchLabels" . | nindent 6 }} 14 | app.kubernetes.io/component: contour 15 | namespaceSelector: 16 | matchNames: 17 | - {{ .Release.Namespace }} 18 | endpoints: 19 | - port: metrics 20 | {{- if .Values.prometheus.serviceMonitor.interval }} 21 | interval: {{ .Values.prometheus.serviceMonitor.interval }} 22 | {{- end }} 23 | {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} 24 | metricRelabelings: {{ toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 4 }} 25 | {{- end }} 26 | {{- if .Values.prometheus.serviceMonitor.relabelings }} 27 | relabelings: {{ toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 4 }} 28 | {{- end }} 29 | {{- end }} 30 | {{- if .Values.envoy.enabled }} 31 | --- 32 | apiVersion: monitoring.coreos.com/v1 33 | kind: ServiceMonitor 34 | metadata: 35 | name: {{ include "contour.fullname" . }}-envoy 36 | labels: {{- include "contour.labels" . | nindent 4}} 37 | app.kubernetes.io/component: envoy 38 | spec: 39 | jobLabel: {{ .Values.prometheus.serviceMonitor.jobLabel | quote }} 40 | selector: 41 | matchLabels: {{- include "contour.matchLabels" . | nindent 6 }} 42 | app.kubernetes.io/component: envoy 43 | namespaceSelector: 44 | matchNames: 45 | - {{ .Release.Namespace }} 46 | endpoints: 47 | - port: metrics 48 | path: /stats/prometheus 49 | {{- if .Values.prometheus.serviceMonitor.interval }} 50 | interval: {{ .Values.prometheus.serviceMonitor.interval }} 51 | {{- end }} 52 | {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} 53 | metricRelabelings: {{ toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 4 }} 54 | {{- end }} 55 | {{- if .Values.prometheus.serviceMonitor.relabelings }} 56 | relabelings: {{ toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 4 }} 57 | {{- end }} 58 | {{- end }} 59 | {{- end }} 60 | -------------------------------------------------------------------------------- /charts/kok-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/kok-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: kok-operator 3 | description: A Helm chart for Kubernetes kok-operator 4 | version: v0.2.0 5 | keywords: 6 | - k8s 7 | - controller 8 | 9 | appVersion: v0.2.0 10 | -------------------------------------------------------------------------------- /charts/kok-operator/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /charts/kok-operator/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "kok-operator.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "kok-operator.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "kok-operator.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "kok-operator.labels" -}} 37 | helm.sh/chart: {{ include "kok-operator.chart" . }} 38 | {{ include "kok-operator.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "kok-operator.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "kok-operator.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "kok-operator.saname" -}} 57 | {{- if .Values.rbac.create }} 58 | {{- default (include "kok-operator.fullname" .) .Values.rbac.name }} 59 | {{- else }} 60 | {{- default "default" .Values.rbac.saname }} 61 | {{- end }} 62 | {{- end }} 63 | 64 | 65 | {{- define "deployment_api_version" -}} 66 | {{- if .Capabilities.APIVersions.Has "apps/v1" -}} 67 | {{- "apps/v1" -}} 68 | {{- else if .Capabilities.APIVersions.Has "apps/v1beta2" -}} 69 | {{- "apps/v1beta1" -}} 70 | {{- else if .Capabilities.APIVersions.Has "apps/v1beta1" -}} 71 | {{- "apps/v1beta1" -}} 72 | {{- else -}} 73 | {{- "extensions/v1beta1" -}} 74 | {{- end -}} 75 | {{- end -}} 76 | 77 | 78 | {{- define "statefulset_api_version" -}} 79 | {{- if .Capabilities.APIVersions.Has "apps/v1" -}} 80 | {{- "apps/v1" -}} 81 | {{- else if .Capabilities.APIVersions.Has "apps/v1beta2" -}} 82 | {{- "apps/v1beta2" -}} 83 | {{- else -}} 84 | {{- "apps/v1beta1" -}} 85 | {{- end -}} 86 | {{- end -}} 87 | 88 | 89 | {{- define "daemonset_api_version" -}} 90 | {{- if .Capabilities.APIVersions.Has "apps/v1" -}} 91 | {{- "apps/v1" -}} 92 | {{- else if .Capabilities.APIVersions.Has "apps/v1beta2" -}} 93 | {{- "apps/v1beta2" -}} 94 | {{- else -}} 95 | {{- "extensions/v1beta1" -}} 96 | {{- end -}} 97 | {{- end -}} 98 | 99 | 100 | {{- define "rbac_api_version" -}} 101 | {{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" -}} 102 | {{- "rbac.authorization.k8s.io/v1" -}} 103 | {{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1beta1" -}} 104 | {{- "rbac.authorization.k8s.io/v1beta1" -}} 105 | {{- else -}} 106 | {{- "rbac.authorization.k8s.io/v1alpha1" -}} 107 | {{- end -}} 108 | {{- end -}} 109 | -------------------------------------------------------------------------------- /charts/kok-operator/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: {{ include "deployment_api_version" . }} 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "kok-operator.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "kok-operator.labels" . | nindent 4 }} 8 | spec: 9 | replicas: {{ .Values.replicaCount }} 10 | selector: 11 | matchLabels: 12 | {{- include "kok-operator.selectorLabels" . | nindent 6 }} 13 | template: 14 | metadata: 15 | {{- with .Values.podAnnotations }} 16 | annotations: 17 | {{- toYaml . | nindent 8 }} 18 | {{- end }} 19 | labels: 20 | {{- include "kok-operator.selectorLabels" . | nindent 8 }} 21 | spec: 22 | {{- with .Values.imagePullSecrets }} 23 | imagePullSecrets: 24 | {{- toYaml . | nindent 8 }} 25 | {{- end }} 26 | serviceAccountName: {{ include "kok-operator.saname" . }} 27 | securityContext: 28 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 29 | containers: 30 | - name: {{ .Chart.Name }} 31 | securityContext: 32 | {{- toYaml .Values.securityContext | nindent 12 }} 33 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 34 | imagePullPolicy: {{ .Values.image.pullPolicy }} 35 | command: 36 | - kok-operator 37 | args: 38 | - ctrl 39 | - -v 40 | - "4" 41 | {{- if .Values.args.imagesPrefix }} 42 | - --images-prefix={{ .Values.args.imagesPrefix }} 43 | {{- end }} 44 | ports: 45 | - name: http 46 | containerPort: 8090 47 | protocol: TCP 48 | # livenessProbe: 49 | # httpGet: 50 | # path: / 51 | # port: http 52 | # readinessProbe: 53 | # httpGet: 54 | # path: / 55 | # port: http 56 | resources: 57 | {{- toYaml .Values.resources | nindent 12 }} 58 | {{- with .Values.nodeSelector }} 59 | nodeSelector: 60 | {{- toYaml . | nindent 8 }} 61 | {{- end }} 62 | {{- with .Values.affinity }} 63 | affinity: 64 | {{- toYaml . | nindent 8 }} 65 | {{- end }} 66 | {{- with .Values.tolerations }} 67 | tolerations: 68 | {{- toYaml . | nindent 8 }} 69 | {{- end }} 70 | -------------------------------------------------------------------------------- /charts/kok-operator/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | --- 3 | apiVersion: {{ include "rbac_api_version" . }} 4 | kind: ClusterRole 5 | metadata: 6 | name: {{ include "kok-operator.saname" . }} 7 | labels: 8 | {{- include "kok-operator.labels" . | nindent 4 }} 9 | rules: 10 | - apiGroups: 11 | - "" 12 | resources: ["*"] 13 | verbs: ["*"] 14 | - apiGroups: 15 | - "coordination.k8s.io" 16 | resources: ["leases"] 17 | verbs: 18 | - create 19 | - get 20 | - list 21 | - update 22 | - apiGroups: 23 | - "apps" 24 | - "apiextensions.k8s.io" 25 | - "autoscaling" 26 | resources: ["*"] 27 | verbs: ["*"] 28 | - apiGroups: ["devops.fake.io","workload.fake.io"] 29 | resources: ["*"] 30 | verbs: ["*"] 31 | --- 32 | apiVersion: v1 33 | kind: ServiceAccount 34 | metadata: 35 | name: {{ include "kok-operator.saname" . }} 36 | namespace: {{ .Release.Namespace }} 37 | labels: 38 | {{- include "kok-operator.labels" . | nindent 4 }} 39 | {{- with .Values.rbac.annotations }} 40 | annotations: 41 | {{- toYaml . | nindent 4 }} 42 | {{- end }} 43 | --- 44 | apiVersion: {{ include "rbac_api_version" . }} 45 | kind: ClusterRoleBinding 46 | metadata: 47 | name: {{ include "kok-operator.saname" . }} 48 | labels: 49 | {{- include "kok-operator.labels" . | nindent 4 }} 50 | subjects: 51 | - kind: ServiceAccount 52 | name: {{ include "kok-operator.saname" . }} 53 | namespace: {{ .Release.Namespace }} 54 | roleRef: 55 | kind: ClusterRole 56 | name: {{ include "kok-operator.saname" . }} 57 | apiGroup: rbac.authorization.k8s.io 58 | --- 59 | {{- end }} 60 | -------------------------------------------------------------------------------- /charts/kok-operator/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "kok-operator.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "kok-operator.labels" . | nindent 4 }} 8 | spec: 9 | type: {{ .Values.service.type }} 10 | ports: 11 | - port: {{ .Values.service.port }} 12 | targetPort: http 13 | protocol: TCP 14 | name: http 15 | selector: 16 | {{- include "kok-operator.selectorLabels" . | nindent 4 }} 17 | -------------------------------------------------------------------------------- /charts/kok-operator/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for kok-operator. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | image: 8 | repository: docker.io/wtxue/kok-operator 9 | # IfNotPresent 10 | pullPolicy: Always 11 | # Overrides the image tag whose default is the chart appVersion. 12 | tag: "v0.2.0" 13 | 14 | args: 15 | imagesPrefix: docker.io/wtxue 16 | 17 | imagePullSecrets: [] 18 | nameOverride: "" 19 | fullnameOverride: "" 20 | 21 | rbac: 22 | # create specifies whether to install and use RBAC rules. 23 | create: true 24 | name: "" 25 | 26 | podAnnotations: {} 27 | 28 | podSecurityContext: {} 29 | # fsGroup: 2000 30 | 31 | securityContext: {} 32 | # capabilities: 33 | # drop: 34 | # - ALL 35 | # readOnlyRootFilesystem: true 36 | # runAsNonRoot: true 37 | # runAsUser: 1000 38 | 39 | service: 40 | type: ClusterIP 41 | port: 80 42 | 43 | 44 | resources: {} 45 | # limits: 46 | # cpu: 100m 47 | # memory: 128Mi 48 | # requests: 49 | # cpu: 100m 50 | # memory: 128Mi 51 | 52 | nodeSelector: {} 53 | 54 | tolerations: [] 55 | 56 | affinity: {} 57 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: Use HostPath for persistent local storage with Kubernetes 3 | name: local-path-provisioner 4 | version: 1.0.0 5 | appVersion: "v0.0.19" 6 | keywords: 7 | - storage 8 | - hostpath 9 | kubeVersion: ">=1.12.0-r0" 10 | home: https://github.com/rancher/local-path-provisioner 11 | sources: 12 | - https://github.com/rancher/local-path-provisioner.git 13 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | You can create a hostpath-backed persistent volume with a persistent volume claim like this: 2 | 3 | apiVersion: v1 4 | kind: PersistentVolumeClaim 5 | metadata: 6 | name: local-path-pvc 7 | spec: 8 | accessModes: 9 | - ReadWriteOnce 10 | storageClassName: {{ .Values.storageClass.name }} 11 | resources: 12 | requests: 13 | storage: 2Gi 14 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "local-path-provisioner.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "local-path-provisioner.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "local-path-provisioner.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "local-path-provisioner.labels" -}} 38 | app.kubernetes.io/name: {{ include "local-path-provisioner.name" . }} 39 | helm.sh/chart: {{ include "local-path-provisioner.chart" . }} 40 | app.kubernetes.io/instance: {{ .Release.Name }} 41 | {{- if .Chart.AppVersion }} 42 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 43 | {{- end }} 44 | app.kubernetes.io/managed-by: {{ .Release.Service }} 45 | {{- end -}} 46 | 47 | {{/* 48 | Create the name of the service account to use. 49 | */}} 50 | {{- define "local-path-provisioner.serviceAccountName" -}} 51 | {{- if .Values.serviceAccount.create -}} 52 | {{ default (include "local-path-provisioner.fullname" .) .Values.serviceAccount.name }} 53 | {{- else -}} 54 | {{ default "default" .Values.serviceAccount.name }} 55 | {{- end -}} 56 | {{- end -}} 57 | 58 | {{/* 59 | Create the name of the provisioner to use. 60 | */}} 61 | {{- define "local-path-provisioner.provisionerName" -}} 62 | {{- if .Values.storageClass.provisionerName -}} 63 | {{- printf .Values.storageClass.provisionerName -}} 64 | {{- else -}} 65 | cluster.local/{{ template "local-path-provisioner.fullname" . -}} 66 | {{- end -}} 67 | {{- end -}} 68 | 69 | {{- define "local-path-provisioner.secret" }} 70 | {{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.privateRegistry.registryUrl (printf "%s:%s" .Values.privateRegistry.registryUser .Values.privateRegistry.registryPasswd | b64enc) | b64enc }} 71 | {{- end }} 72 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: {{ include "local-path-provisioner.fullname" . }} 6 | labels: 7 | {{ include "local-path-provisioner.labels" . | indent 4 }} 8 | rules: 9 | - apiGroups: [""] 10 | resources: ["nodes", "persistentvolumeclaims", "configmaps"] 11 | verbs: ["get", "list", "watch"] 12 | - apiGroups: [""] 13 | resources: ["endpoints", "persistentvolumes", "pods"] 14 | verbs: ["*"] 15 | - apiGroups: [""] 16 | resources: ["events"] 17 | verbs: ["create", "patch"] 18 | - apiGroups: ["storage.k8s.io"] 19 | resources: ["storageclasses"] 20 | verbs: ["get", "list", "watch"] 21 | {{- end -}} 22 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: {{ include "local-path-provisioner.fullname" . }} 6 | labels: 7 | {{ include "local-path-provisioner.labels" . | indent 4 }} 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: ClusterRole 11 | name: {{ template "local-path-provisioner.fullname" . }} 12 | subjects: 13 | - kind: ServiceAccount 14 | name: {{ template "local-path-provisioner.serviceAccountName" . }} 15 | namespace: {{ .Release.Namespace }} 16 | {{- end -}} 17 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ .Values.configmap.name }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{ include "local-path-provisioner.labels" . | indent 4 }} 8 | data: 9 | config.json: |- 10 | { 11 | "nodePathMap": {{ .Values.nodePathMap | toPrettyJson | nindent 8 }} 12 | } 13 | setup: |- 14 | {{ .Values.configmap.setup | nindent 4 }} 15 | teardown: |- 16 | {{ .Values.configmap.teardown | nindent 4 }} 17 | helperPod.yaml: |- 18 | {{ .Values.configmap.helperPod | nindent 4 }} 19 | 20 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "local-path-provisioner.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{ include "local-path-provisioner.labels" . | indent 4 }} 8 | spec: 9 | replicas: {{ .Values.replicaCount }} 10 | selector: 11 | matchLabels: 12 | app.kubernetes.io/name: {{ include "local-path-provisioner.name" . }} 13 | app.kubernetes.io/instance: {{ .Release.Name }} 14 | template: 15 | metadata: 16 | labels: 17 | app.kubernetes.io/name: {{ include "local-path-provisioner.name" . }} 18 | app.kubernetes.io/instance: {{ .Release.Name }} 19 | spec: 20 | {{- with .Values.imagePullSecrets }} 21 | imagePullSecrets: 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | serviceAccountName: {{ template "local-path-provisioner.serviceAccountName" . }} 25 | containers: 26 | - name: {{ .Chart.Name }} 27 | {{- if .Values.privateRegistry.registryUrl }} 28 | image: "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 29 | {{- else }} 30 | image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 31 | {{- end }} 32 | imagePullPolicy: {{ .Values.image.pullPolicy }} 33 | command: 34 | - local-path-provisioner 35 | - --debug 36 | - start 37 | - --config 38 | - /etc/config/config.json 39 | - --service-account-name 40 | - {{ template "local-path-provisioner.serviceAccountName" . }} 41 | - --provisioner-name 42 | - {{ template "local-path-provisioner.provisionerName" . }} 43 | - --helper-image 44 | {{- if .Values.privateRegistry.registryUrl }} 45 | - "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.helperImage.repository }}:{{ .Values.helperImage.tag }}" 46 | {{- else }} 47 | - "{{ .Values.helperImage.repository }}:{{ .Values.helperImage.tag }}" 48 | {{- end }} 49 | - --configmap-name 50 | - {{ .Values.configmap.name }} 51 | volumeMounts: 52 | - name: config-volume 53 | mountPath: /etc/config/ 54 | env: 55 | - name: POD_NAMESPACE 56 | value: {{ .Release.Namespace }} 57 | resources: 58 | {{- toYaml .Values.resources | nindent 12 }} 59 | volumes: 60 | - name: config-volume 61 | configMap: 62 | name: {{ .Values.configmap.name }} 63 | {{- with .Values.nodeSelector }} 64 | nodeSelector: 65 | {{- toYaml . | nindent 8 }} 66 | {{- end }} 67 | {{- with .Values.affinity }} 68 | affinity: 69 | {{- toYaml . | nindent 8 }} 70 | {{- end }} 71 | {{- with .Values.tolerations }} 72 | tolerations: 73 | {{- toYaml . | nindent 8 }} 74 | {{- end }} 75 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/registry-secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.defaultSettings.registrySecret }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ .Values.defaultSettings.registrySecret }} 6 | namespace: {{ .Release.Namespace }} 7 | type: kubernetes.io/dockerconfigjson 8 | data: 9 | .dockerconfigjson: {{ template "local-path-provisioner.secret" . }} 10 | {{- end }} 11 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "local-path-provisioner.serviceAccountName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{ include "local-path-provisioner.labels" . | indent 4 }} 9 | {{- with .Values.imagePullSecrets }} 10 | imagePullSecrets: 11 | {{- toYaml . | nindent 2 }} 12 | {{- end }} 13 | {{- if .Values.defaultSettings.registrySecret }} 14 | - name: {{ .Values.defaultSettings.registrySecret }} 15 | {{- end }} 16 | {{- end }} 17 | -------------------------------------------------------------------------------- /charts/local-path-provisioner/templates/storageclass.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.storageClass.create -}} 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: {{ .Values.storageClass.name }} 6 | labels: 7 | {{ include "local-path-provisioner.labels" . | indent 4 }} 8 | {{- if .Values.storageClass.defaultClass }} 9 | annotations: 10 | storageclass.kubernetes.io/is-default-class: "true" 11 | {{- end }} 12 | provisioner: {{ template "local-path-provisioner.provisionerName" . }} 13 | volumeBindingMode: WaitForFirstConsumer 14 | reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} 15 | {{- end }} 16 | -------------------------------------------------------------------------------- /cmd/controller/app/app_option/option.go: -------------------------------------------------------------------------------- 1 | package app_option 2 | 3 | import ( 4 | "github.com/spf13/pflag" 5 | "github.com/wtxue/kok-operator/pkg/option" 6 | "github.com/wtxue/kok-operator/pkg/provider/config" 7 | ) 8 | 9 | // Options ... 10 | type Options struct { 11 | Global *option.GlobalManagerOption 12 | Ctrl *option.ControllersManagerOption 13 | Provider *config.Config 14 | } 15 | 16 | // NewOptions creates a new Options with a default config. 17 | func NewOptions() *Options { 18 | return &Options{ 19 | Global: option.DefaultGlobalManagetOption(), 20 | Ctrl: option.DefaultControllersManagerOption(), 21 | Provider: config.NewDefaultConfig(), 22 | } 23 | } 24 | 25 | // AddFlags adds flags for a specific server to the specified FlagSet object. 26 | func (o *Options) AddFlags(fs *pflag.FlagSet) { 27 | o.Global.AddFlags(fs) 28 | o.Ctrl.AddFlags(fs) 29 | o.Provider.AddFlags(fs) 30 | } 31 | -------------------------------------------------------------------------------- /cmd/controller/app/certs.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "crypto" 5 | "crypto/x509" 6 | "net" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/spf13/cobra" 10 | "github.com/wtxue/kok-operator/cmd/controller/app/app_option" 11 | "github.com/wtxue/kok-operator/pkg/util/pkiutil" 12 | certutil "k8s.io/client-go/util/cert" 13 | "k8s.io/klog/v2" 14 | ) 15 | 16 | func writeCertificateAuthorityFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey crypto.Signer) error { 17 | // If cert or key exists, we should try to load them 18 | if pkiutil.CertOrKeyExist(pkiDir, baseName) { 19 | 20 | // Try to load .crt and .key from the PKI directory 21 | caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) 22 | if err != nil { 23 | return errors.Wrapf(err, "failure loading %s certificate", baseName) 24 | } 25 | 26 | // Check if the existing cert is a CA 27 | if !caCert.IsCA { 28 | return errors.Errorf("certificate %s is not a CA", baseName) 29 | } 30 | 31 | // kubeadm doesn't validate the existing certificate Authority more than this; 32 | // Basically, if we find a certificate file with the same path; and it is a CA 33 | // kubeadm thinks those files are equal and doesn't bother writing a new file 34 | klog.Infof("[certs] Using the existing %s certificate and key\n", baseName) 35 | } else { 36 | // Write .crt and .key files to disk 37 | klog.Infof("[certs] Generating %s certificate and key\n", baseName) 38 | 39 | if err := pkiutil.WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil { 40 | return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) 41 | } 42 | } 43 | return nil 44 | } 45 | 46 | func TryRun() error { 47 | klog.Infof("try run cert ... ") 48 | 49 | klog.Infof("start generating ca certificate and key ... ") 50 | caCsr := &pkiutil.CertConfig{ 51 | PublicKeyAlgorithm: x509.RSA, 52 | Config: certutil.Config{ 53 | CommonName: "kubernetes", 54 | Organization: []string{"k8s"}, 55 | Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, 56 | }, 57 | } 58 | 59 | caCert, caKey, err := pkiutil.NewCertificateAuthority(caCsr) 60 | if err != nil { 61 | return err 62 | } 63 | klog.Infof("start generating ca certificate and key success ") 64 | 65 | apiserverCsr := &pkiutil.CertConfig{ 66 | PublicKeyAlgorithm: x509.RSA, 67 | Config: certutil.Config{ 68 | CommonName: "kube-apiserver", 69 | Organization: []string{"dke"}, 70 | AltNames: certutil.AltNames{ 71 | DNSNames: []string{ 72 | "vip-otdyiqyb.dke.k8s.io", // dns 73 | "localhost", 74 | "kubernetes", 75 | "kubernetes.default", 76 | "kubernetes.default.svc", 77 | "kubernetes.default.svc.cluster", 78 | "kubernetes.default.svc.cluster.local", 79 | }, 80 | IPs: []net.IP{ 81 | net.ParseIP("127.0.0.1"), // 82 | net.ParseIP("10.12.99.10"), // master1 83 | net.ParseIP("10.12.99.10"), // master2 84 | net.ParseIP("10.12.99.10"), // master3 85 | net.ParseIP("10.12.99.254"), // vip 86 | net.ParseIP("10.96.0.1"), // kubernetes svc ip, always first svc cidr 87 | }, 88 | }, 89 | Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, 90 | }, 91 | } 92 | 93 | pkiutil.NewCertAndKey(caCert, caKey, apiserverCsr) 94 | writeCertificateAuthorityFilesIfNotExist("./tools/pki", "ca", caCert, caKey) 95 | return nil 96 | } 97 | 98 | func NewCertCmd(opt *app_option.Options) *cobra.Command { 99 | cmd := &cobra.Command{ 100 | Use: "cert", 101 | Short: "Manage k8s cert demo", 102 | Run: func(cmd *cobra.Command, args []string) { 103 | opt.Global.SetupLogger() 104 | TryRun() 105 | }, 106 | } 107 | 108 | return cmd 109 | } 110 | -------------------------------------------------------------------------------- /cmd/controller/app/controller.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/wtxue/kok-operator/cmd/controller/app/app_option" 6 | "github.com/wtxue/kok-operator/pkg/controllers" 7 | "github.com/wtxue/kok-operator/pkg/k8sclient" 8 | "github.com/wtxue/kok-operator/pkg/k8sutil" 9 | "github.com/wtxue/kok-operator/pkg/static" 10 | "k8s.io/klog/v2" 11 | ctrlrt "sigs.k8s.io/controller-runtime" 12 | "sigs.k8s.io/controller-runtime/pkg/manager" 13 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 14 | ) 15 | 16 | func NewControllerCmd(opt *app_option.Options) *cobra.Command { 17 | cmd := &cobra.Command{ 18 | Use: "ctrl", 19 | Short: "Manage controller Component", 20 | Run: func(cmd *cobra.Command, args []string) { 21 | PrintFlags(cmd.Flags()) 22 | opt.Global.SetupLogger() 23 | 24 | cfg, err := opt.Global.GetK8sConfig() 25 | if err != nil { 26 | klog.Fatalf("unable to get cfg err: %v", err) 27 | } 28 | 29 | if opt.Ctrl.EnableManagerCrds { 30 | crds, err := static.LoadCRDs() 31 | if err != nil { 32 | klog.Fatalf("unable to get cfg err: %v", err) 33 | } 34 | 35 | err = k8sutil.ReconcileCRDs(cfg, crds) 36 | if err != nil { 37 | klog.Fatalf("failed to reconcile crd err: %v", err) 38 | } 39 | } 40 | 41 | // Adjust our client's rate limits based on the number of controllers we are running. 42 | cfg.QPS = float32(2) * cfg.QPS 43 | cfg.Burst = 2 * cfg.Burst 44 | 45 | mgr, err := manager.New(cfg, manager.Options{ 46 | Scheme: k8sclient.GetScheme(), 47 | LeaderElection: opt.Global.EnableLeaderElection, 48 | LeaderElectionResourceLock: "leases", 49 | LeaderElectionNamespace: opt.Global.LeaderElectionNamespace, 50 | LeaderElectionID: "kok-operator", 51 | SyncPeriod: &opt.Global.ResyncPeriod, 52 | MetricsBindAddress: "0", // disable metrics with manager, use our observe 53 | HealthProbeBindAddress: "0", // disable health probe with manager, use our observe 54 | }) 55 | if err != nil { 56 | klog.Fatalf("unable to new manager err: %v", err) 57 | } 58 | 59 | // Setup all Controllers 60 | ctrlrt.Log.Info("Setting up controller") 61 | if err := controllers.AddToManager(mgr, opt.Ctrl, opt.Provider); err != nil { 62 | klog.Fatalf("unable to register controllers to the manager err: %v", err) 63 | } 64 | 65 | ctrlrt.Log.Info("starting manager") 66 | stopCh := signals.SetupSignalHandler() 67 | if err := mgr.Start(stopCh); err != nil { 68 | klog.Fatalf("problem start running manager err: %v", err) 69 | } 70 | }, 71 | } 72 | 73 | opt.Ctrl.AddFlags(cmd.Flags()) 74 | opt.Provider.AddFlags(cmd.Flags()) 75 | return cmd 76 | } 77 | -------------------------------------------------------------------------------- /cmd/controller/app/fake_apiserver.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/wtxue/kok-operator/cmd/controller/app/app_option" 10 | "github.com/wtxue/kok-operator/pkg/apiserver" 11 | "github.com/wtxue/kok-operator/pkg/option" 12 | "k8s.io/client-go/rest" 13 | "k8s.io/client-go/tools/clientcmd" 14 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 15 | "k8s.io/klog/v2" 16 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 17 | ) 18 | 19 | // CreateKubeConfigFile creates a kubeconfig file. 20 | func CreateKubeConfigFile(outDir string, kubeConfigFileName string, cfg *rest.Config) error { 21 | klog.Infof("creating kubeconfig file for %s", kubeConfigFileName) 22 | 23 | clusterName := "fake-cluster" 24 | userName := "devops" 25 | 26 | contextName := fmt.Sprintf("%s@%s", userName, clusterName) 27 | apiConfig := &clientcmdapi.Config{ 28 | Clusters: map[string]*clientcmdapi.Cluster{ 29 | clusterName: { 30 | Server: cfg.Host, 31 | }, 32 | }, 33 | Contexts: map[string]*clientcmdapi.Context{ 34 | contextName: { 35 | Cluster: clusterName, 36 | AuthInfo: userName, 37 | }, 38 | }, 39 | AuthInfos: map[string]*clientcmdapi.AuthInfo{}, 40 | CurrentContext: contextName, 41 | } 42 | 43 | apiConfig.AuthInfos[userName] = &clientcmdapi.AuthInfo{} 44 | 45 | kubeConfigFilePath := filepath.Join(outDir, kubeConfigFileName) 46 | err := clientcmd.WriteToFile(*apiConfig, kubeConfigFilePath) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | klog.Infof("kubeconfig file for [%s@%s] is write to path: %s", clusterName, userName, kubeConfigFilePath) 52 | return nil 53 | } 54 | 55 | func tryRun(opt *option.ApiServerOption, ctx context.Context) error { 56 | if opt.IsLocalKube { 57 | svc := apiserver.New(opt) 58 | cfg, err := svc.Start(ctx.Done()) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | err = CreateKubeConfigFile(opt.RootDir+"/cfg", "fake-kubeconfig.yaml", cfg) 64 | if err != nil { 65 | return err 66 | } 67 | } 68 | 69 | return nil 70 | } 71 | 72 | func NewFakeApiserverCmd(opt *app_option.Options) *cobra.Command { 73 | var isStart bool 74 | apiServerOpt := option.DefaultApiServerOption() 75 | 76 | cmd := &cobra.Command{ 77 | Use: "fake", 78 | Short: "Manage with a fake apiserver", 79 | Run: func(cmd *cobra.Command, args []string) { 80 | opt.Global.SetupLogger() 81 | ctx := signals.SetupSignalHandler() 82 | err := tryRun(apiServerOpt, ctx) 83 | if err != nil { 84 | klog.Errorf("err: %+v", err) 85 | return 86 | } 87 | 88 | <-ctx.Done() 89 | klog.Infof("stop fake api server") 90 | }, 91 | } 92 | 93 | cmd.Flags().BoolVarP(&isStart, "start", "s", true, "Enables start fake apiserver") 94 | apiServerOpt.AddFlags(cmd.Flags()) 95 | return cmd 96 | } 97 | -------------------------------------------------------------------------------- /cmd/controller/app/root.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/spf13/cobra" 7 | "github.com/spf13/pflag" 8 | "github.com/wtxue/kok-operator/cmd/controller/app/app_option" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | func AddFlags(cmd *cobra.Command) { 13 | cmd.PersistentFlags().AddGoFlagSet(flag.CommandLine) 14 | } 15 | 16 | // PrintFlags logs the flags in the flagset 17 | func PrintFlags(flags *pflag.FlagSet) { 18 | flags.VisitAll(func(flag *pflag.Flag) { 19 | klog.Infof("FLAG: --%s=%q", flag.Name, flag.Value) 20 | }) 21 | } 22 | 23 | func runHelp(cmd *cobra.Command, args []string) { 24 | cmd.Help() 25 | } 26 | 27 | // GetRootCmd returns the root of the cobra command-tree. 28 | func GetRootCmd(args []string) *cobra.Command { 29 | opt := app_option.NewOptions() 30 | rootCmd := &cobra.Command{ 31 | Use: "kok-operator", 32 | Short: "Request a new kok operator", 33 | SilenceUsage: true, 34 | DisableAutoGenTag: true, 35 | Run: runHelp, 36 | } 37 | 38 | rootCmd.SetArgs(args) 39 | opt.Global.AddFlags(rootCmd.PersistentFlags()) 40 | 41 | // Make sure that klog logging variables are initialized so that we can 42 | // update them from this file. 43 | klog.InitFlags(nil) 44 | 45 | // Make sure klog (used by the client-go dependency) logs to stderr, as it 46 | // will try to log to directories that may not exist in the cilium-operator 47 | // container (/tmp) and cause the cilium-operator to exit. 48 | flag.Set("logtostderr", "true") 49 | AddFlags(rootCmd) 50 | 51 | rootCmd.AddCommand(NewControllerCmd(opt)) 52 | rootCmd.AddCommand(NewFakeApiserverCmd(opt)) 53 | rootCmd.AddCommand(NewCertCmd(opt)) 54 | rootCmd.AddCommand(NewCmdVersion()) 55 | return rootCmd 56 | } 57 | 58 | func hideInheritedFlags(orig *cobra.Command, hidden ...string) { 59 | orig.SetHelpFunc(func(cmd *cobra.Command, args []string) { 60 | for _, hidden := range hidden { 61 | _ = cmd.Flags().MarkHidden(hidden) // nolint: errcheck 62 | } 63 | 64 | orig.SetHelpFunc(nil) 65 | orig.HelpFunc()(cmd, args) 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /cmd/controller/app/version.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | "github.com/wtxue/kok-operator/pkg/version" 9 | ) 10 | 11 | // NewCmdVersion returns a cobra command for fetching versions 12 | func NewCmdVersion() *cobra.Command { 13 | cmd := &cobra.Command{ 14 | Use: "version", 15 | Short: "Print the version information", 16 | Long: "Print the version information for the current context", 17 | Run: func(cmd *cobra.Command, args []string) { 18 | v := version.GetVersion() 19 | fmt.Fprintf(os.Stdout, "version: %v\n", v.String()) 20 | }, 21 | } 22 | 23 | return cmd 24 | } 25 | -------------------------------------------------------------------------------- /cmd/controller/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 wtxue. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "fmt" 21 | "math/rand" 22 | "os" 23 | "time" 24 | 25 | "github.com/wtxue/kok-operator/cmd/controller/app" 26 | 27 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 28 | ) 29 | 30 | func main() { 31 | rand.Seed(time.Now().UnixNano()) 32 | 33 | rootCmd := app.GetRootCmd(os.Args[1:]) 34 | 35 | if err := rootCmd.Execute(); err != nil { 36 | fmt.Fprintf(os.Stderr, "error: %v\n", err) 37 | os.Exit(-1) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /config/crd/bases/devops.fake.io_clustercredentials.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.5.0 8 | creationTimestamp: null 9 | name: clustercredentials.devops.fake.io 10 | spec: 11 | group: devops.fake.io 12 | names: 13 | kind: ClusterCredential 14 | listKind: ClusterCredentialList 15 | plural: clustercredentials 16 | singular: clustercredential 17 | scope: Namespaced 18 | versions: 19 | - name: v1 20 | schema: 21 | openAPIV3Schema: 22 | description: ClusterCredential records the credential information needed to access the cluster. 23 | properties: 24 | apiVersion: 25 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 26 | type: string 27 | bootstrapToken: 28 | description: For kubeadm init or join 29 | type: string 30 | caCert: 31 | description: For connect the cluster 32 | format: byte 33 | type: string 34 | caKey: 35 | format: byte 36 | type: string 37 | certificateKey: 38 | description: For kubeadm init or join 39 | type: string 40 | certsBinaryData: 41 | additionalProperties: 42 | format: byte 43 | type: string 44 | type: object 45 | clientCert: 46 | description: For kube-apiserver X509 auth 47 | format: byte 48 | type: string 49 | clientKey: 50 | description: For kube-apiserver X509 auth 51 | format: byte 52 | type: string 53 | clusterName: 54 | type: string 55 | etcdAPIClientCert: 56 | format: byte 57 | type: string 58 | etcdAPIClientKey: 59 | format: byte 60 | type: string 61 | etcdCACert: 62 | description: For TKE in global reuse 63 | format: byte 64 | type: string 65 | etcdCAKey: 66 | format: byte 67 | type: string 68 | extData: 69 | additionalProperties: 70 | type: string 71 | type: object 72 | kind: 73 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 74 | type: string 75 | kubeData: 76 | additionalProperties: 77 | type: string 78 | type: object 79 | manifestsData: 80 | additionalProperties: 81 | type: string 82 | type: object 83 | metadata: 84 | type: object 85 | tenantID: 86 | type: string 87 | token: 88 | description: For kube-apiserver token auth 89 | type: string 90 | required: 91 | - clusterName 92 | - tenantID 93 | type: object 94 | served: true 95 | storage: true 96 | status: 97 | acceptedNames: 98 | kind: "" 99 | plural: "" 100 | conditions: [] 101 | storedVersions: [] 102 | -------------------------------------------------------------------------------- /config/crd/bases/workload.fake.io_addons.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.5.0 8 | creationTimestamp: null 9 | name: addons.workload.fake.io 10 | spec: 11 | group: workload.fake.io 12 | names: 13 | kind: Addons 14 | listKind: AddonsList 15 | plural: addons 16 | singular: addons 17 | scope: Namespaced 18 | versions: 19 | - additionalPrinterColumns: 20 | - description: The Addons phase. 21 | jsonPath: .status.phase 22 | name: PHASE 23 | type: string 24 | - description: 'CreationTimestamp is a timestamp representing the server time when this object was created. ' 25 | jsonPath: .metadata.creationTimestamp 26 | name: AGE 27 | type: date 28 | name: v1 29 | schema: 30 | openAPIV3Schema: 31 | description: Addons is the Schema for the Addon API 32 | properties: 33 | apiVersion: 34 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 35 | type: string 36 | kind: 37 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 38 | type: string 39 | metadata: 40 | type: object 41 | spec: 42 | description: AddonsSpec is a description of Addons. 43 | properties: 44 | foo: 45 | type: string 46 | type: object 47 | status: 48 | description: AddonsStatus represents information about the status of an Addons. 49 | properties: 50 | foo: 51 | type: string 52 | phase: 53 | type: string 54 | type: object 55 | type: object 56 | served: true 57 | storage: true 58 | subresources: 59 | status: {} 60 | status: 61 | acceptedNames: 62 | kind: "" 63 | plural: "" 64 | conditions: [] 65 | storedVersions: [] 66 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - apiGroups: 10 | - devops.fake.io 11 | resources: 12 | - clusters 13 | verbs: 14 | - create 15 | - delete 16 | - get 17 | - list 18 | - patch 19 | - update 20 | - watch 21 | - apiGroups: 22 | - devops.fake.io 23 | resources: 24 | - clusters/status 25 | verbs: 26 | - get 27 | - patch 28 | - update 29 | - apiGroups: 30 | - devops.fake.io 31 | resources: 32 | - virtulclusters 33 | verbs: 34 | - create 35 | - delete 36 | - get 37 | - list 38 | - patch 39 | - update 40 | - watch 41 | - apiGroups: 42 | - devops.fake.io 43 | resources: 44 | - virtulclusters/status 45 | verbs: 46 | - get 47 | - patch 48 | - update 49 | -------------------------------------------------------------------------------- /doc/addons.md: -------------------------------------------------------------------------------- 1 | # kube-prometheus 2 | 3 | ~~~ shell 4 | # add helm repo 5 | helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 6 | helm repo add stable https://charts.helm.sh/stable 7 | helm repo update 8 | 9 | # custom helm install override values 10 | cat > ./overridevalues-kube-prometheus.yaml < ./overridevalues-loki.yaml < /etc/containerd/config.toml 11 | 12 | 13 | docker.io/rancher/pause:3.2 14 | k8s.gcr.io/pause:3.2 15 | 16 | 17 | # mirrors and config 18 | [plugins] 19 | [plugins."io.containerd.grpc.v1.cri"] 20 | sandbox_image = "docker.io/rancher/pause:3.2" 21 | [plugins."io.containerd.grpc.v1.cri".registry] 22 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors] 23 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] 24 | endpoint = [ 25 | "https://yqdzw3p0.mirror.aliyuncs.com" 26 | ] 27 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"] 28 | endpoint = [ 29 | "https://registry.aliyuncs.com/k8sxio" 30 | ] 31 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"] 32 | endpoint = [ 33 | "https://quay.mirrors.ustc.edu.cn" 34 | ] 35 | 36 | [plugins."io.containerd.grpc.v1.cri".registry] 37 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors] 38 | ... 39 | [plugins."io.containerd.grpc.v1.cri".registry.configs] 40 | [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.example.net".tls] 41 | insecure_skip_verify = true 42 | [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.example.net".auth] 43 | username = "admin" 44 | password = "Mydlq123456" 45 | 46 | ~~~ 47 | 48 | # Containerd + Docker 49 | 50 | 事实上，Docker 和 Containerd 是可以同时使用的，只不过 Docker 默认使用的 Containerd 的命名空间不是 default，而是 `moby`。 首先从其他装了 Docker 的机器或者 GitHub 上下载 Docker 51 | 相关的二进制文件，然后使用下面的命令启动 Docker： 52 | 53 | ~~~shell 54 | dockerd --containerd /run/containerd/containerd.sock --cri-containerd 55 | ctr ns ls 56 | ctr -n moby c ls 57 | ~~~ 58 | -------------------------------------------------------------------------------- /docker/base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stable-20220822 2 | 3 | RUN apt-get update && \ 4 | apt-get install -y procps dnsutils ca-certificates curl git openssl unzip && \ 5 | apt-get install -y net-tools tcpdump ipvsadm telnet iotop wget iptables dnsutils && \ 6 | apt-get autoremove -y && \ 7 | apt-get clean -y 8 | 9 | RUN curl -fsSL https://hey-release.s3.us-east-2.amazonaws.com/hey_linux_amd64 -o hey \ 10 | && chmod a+x hey && mv hey /usr/local/bin/ 11 | -------------------------------------------------------------------------------- /docker/coredns/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG CENTOS_BASE_VERSION=8.3 2 | ARG COREDNS_BASE_VERSION=1.7.0 3 | 4 | FROM coredns/coredns:${COREDNS_BASE_VERSION} as default 5 | 6 | FROM wtxue/centos-base:${CENTOS_BASE_VERSION} 7 | 8 | COPY --from=default /etc/ssl/certs /etc/ssl/certs 9 | COPY --from=default /coredns /coredns 10 | 11 | EXPOSE 53 53/udp 12 | ENTRYPOINT ["/coredns"] 13 | 14 | 15 | -------------------------------------------------------------------------------- /docker/kok-base/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_VERSION=stable-20220822 2 | 3 | ARG K8S_ETCD_VERSION2=3.5.3-0 4 | 5 | FROM registry.aliyuncs.com/google_containers/etcd:${K8S_ETCD_VERSION2} as etcd-v2 6 | FROM docker.io/wtxue/debian-base:${BASE_VERSION} as default 7 | 8 | ENV K9S_SERVER_VERSION v0.26.3 9 | RUN curl -fsSL https://github.com/derailed/k9s/releases/download/$K9S_SERVER_VERSION/k9s_Linux_x86_64.tar.gz -o k9s.tar.gz \ 10 | && tar -C /tmp/ -xzf k9s.tar.gz \ 11 | && mkdir -p /k8s/bin/ && mv /tmp/k9s /k8s/bin/ 12 | 13 | ENV CNI_PLUGINS_VERSION v1.1.1 14 | RUN curl -fsSL https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGINS_VERSION/cni-plugins-linux-amd64-$CNI_PLUGINS_VERSION.tgz -o cni.tgz \ 15 | && mkdir -p /k8s/bin/ && mv cni.tgz /k8s/bin/ 16 | 17 | 18 | ENV CRICTL_VERSION="v1.25.0" 19 | RUN curl -fsSL https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz -o crictl-linux-amd64.tar.gz \ 20 | && tar zxvf crictl-linux-amd64.tar.gz -C /k8s/bin/ 21 | 22 | ENV RUNC_VERSION="v1.1.4" 23 | RUN curl -fsSL https://github.com/opencontainers/runc/releases/download/$RUNC_VERSION/runc.amd64 -o runc \ 24 | && mv runc /k8s/bin/ 25 | 26 | ENV Containerd_VERSION 1.6.8 27 | RUN curl -fsSL https://github.com/containerd/containerd/releases/download/v${Containerd_VERSION}/cri-containerd-cni-${Containerd_VERSION}-linux-amd64.tar.gz -o containerd.tar.gz \ 28 | && mkdir -p /k8s/bin/ && mv containerd.tar.gz /k8s/bin/ 29 | 30 | ENV HELM_VERSION v3.9.4 31 | RUN curl -fsSL https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz -o helm.tar.gz \ 32 | && tar -C /tmp/ -xzf helm.tar.gz \ 33 | && mkdir -p /k8s/bin/ && mv /tmp/linux-amd64/helm /k8s/bin/ 34 | 35 | ENV K8S_V2 v1.24.4 36 | RUN curl -fsSL https://dl.k8s.io/${K8S_V2}/kubernetes-server-linux-amd64.tar.gz -o k8s-${K8S_V2}.tar.gz 37 | RUN mkdir -p /k8s-${K8S_V2}/bin/ && tar -C /k8s-${K8S_V2} -xf k8s-${K8S_V2}.tar.gz \ 38 | && mv /k8s-${K8S_V2}/kubernetes/server/bin/kube-apiserver /k8s-${K8S_V2}/bin/ \ 39 | && mv /k8s-${K8S_V2}/kubernetes/server/bin/kubeadm /k8s-${K8S_V2}/bin/ \ 40 | && mv /k8s-${K8S_V2}/kubernetes/server/bin/kubectl /k8s-${K8S_V2}/bin/ \ 41 | && mv /k8s-${K8S_V2}/kubernetes/server/bin/kubelet /k8s-${K8S_V2}/bin/ \ 42 | && mv /k8s-${K8S_V2}/kubernetes/server/bin/kube-scheduler /k8s-${K8S_V2}/bin/ \ 43 | && mv /k8s-${K8S_V2}/kubernetes/server/bin/kube-controller-manager /k8s-${K8S_V2}/bin/ 44 | 45 | 46 | COPY --from=etcd-v2 /usr/local/bin/etcd \ 47 | /usr/local/bin/etcdctl \ 48 | /k8s-$K8S_V2/bin/ 49 | 50 | FROM docker.io/wtxue/debian-base:${BASE_VERSION} 51 | 52 | ENV K8S_V2 v1.24.4 53 | COPY --from=default /k8s-${K8S_V2}/bin /k8s-${K8S_V2}/bin 54 | 55 | COPY --from=default /k8s/bin /k8s/bin 56 | 57 | -------------------------------------------------------------------------------- /docker/kok-operator/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG KOK_BASE_VERSION=v1.24.4 2 | 3 | FROM docker.io/wtxue/kok-base:${KOK_BASE_VERSION} 4 | 5 | COPY bin/kok-operator /usr/local/bin/ 6 | 7 | 8 | -------------------------------------------------------------------------------- /docker/kubernetes/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_VERSION=stable-20210408 2 | ARG ETCD_BASE_VERSION=v3.4.15 3 | 4 | FROM quay.io/coreos/etcd:${ETCD_BASE_VERSION} as etcd-default 5 | FROM docker.io/wtxue/debian-base:${BASE_VERSION} as default 6 | ENV K8S_SERVER_VERSION v1.20.6 7 | RUN curl -fsSL https://dl.k8s.io/$K8S_SERVER_VERSION/kubernetes-server-linux-amd64.tar.gz -o kubernetes-server-linux-amd64.tar.gz \ 8 | && tar -C /usr/local -xzf kubernetes-server-linux-amd64.tar.gz 9 | 10 | FROM docker.io/wtxue/debian-base:${BASE_VERSION} 11 | 12 | COPY --from=etcd-default /usr/local/bin/etcd \ 13 | /usr/local/bin/etcdctl \ 14 | /usr/local/bin/ 15 | 16 | COPY --from=default /usr/local/kubernetes/server/bin/kube-apiserver \ 17 | /usr/local/kubernetes/server/bin/kubeadm \ 18 | /usr/local/kubernetes/server/bin/kubectl \ 19 | /usr/local/kubernetes/server/bin/kubelet \ 20 | /usr/local/kubernetes/server/bin/kube-scheduler \ 21 | /usr/local/kubernetes/server/bin/kube-controller-manager \ 22 | /usr/local/bin/ 23 | -------------------------------------------------------------------------------- /docker/metrics-server/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG CENTOS_BASE_VERSION=8.3 2 | ARG METRICS_SERVER_BASE_VERSION=v0.3.6 3 | 4 | FROM k8s.gcr.io/metrics-server:${METRICS_SERVER_BASE_VERSION} as default 5 | 6 | FROM wtxue/centos-base:${CENTOS_BASE_VERSION} 7 | 8 | COPY --from=default /metrics-server / 9 | 10 | USER 65534 11 | 12 | ENTRYPOINT ["/metrics-server"] 13 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 wtxue. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /manifests/addons/kube-vip-243.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | creationTimestamp: null 5 | name: kube-vip 6 | namespace: kube-system 7 | spec: 8 | containers: 9 | - args: 10 | - start 11 | env: 12 | - name: vip_arp 13 | value: "true" 14 | - name: vip_interface 15 | value: ens34 16 | - name: vip_leaderelection 17 | value: "true" 18 | - name: vip_leaseduration 19 | value: "5" 20 | - name: vip_renewdeadline 21 | value: "3" 22 | - name: vip_retryperiod 23 | value: "1" 24 | - name: vip_address 25 | value: 172.16.18.243 26 | image: plndr/kube-vip:0.2.3 27 | imagePullPolicy: Always 28 | name: kube-vip 29 | resources: {} 30 | volumeMounts: 31 | - mountPath: /etc/kubernetes/ 32 | name: kubeconfig 33 | readOnly: true 34 | securityContext: 35 | capabilities: 36 | add: 37 | - NET_ADMIN 38 | - SYS_TIME 39 | hostNetwork: true 40 | dnsPolicy: ClusterFirstWithHostNet 41 | volumes: 42 | - hostPath: 43 | path: /etc/kubernetes/ 44 | type: DirectoryOrCreate 45 | name: kubeconfig 46 | status: {} 47 | -------------------------------------------------------------------------------- /manifests/addons/kube-vip.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | creationTimestamp: null 5 | name: kube-vip 6 | namespace: kube-system 7 | spec: 8 | containers: 9 | - args: 10 | - start 11 | env: 12 | - name: vip_arp 13 | value: "true" 14 | - name: vip_interface 15 | value: ens34 16 | - name: vip_leaderelection 17 | value: "true" 18 | - name: vip_leaseduration 19 | value: "5" 20 | - name: vip_renewdeadline 21 | value: "3" 22 | - name: vip_retryperiod 23 | value: "1" 24 | - name: vip_address 25 | value: 172.16.18.241 26 | image: plndr/kube-vip:0.2.3 27 | imagePullPolicy: Always 28 | name: kube-vip 29 | resources: { } 30 | volumeMounts: 31 | - mountPath: /etc/kubernetes/ 32 | name: kubeconfig 33 | readOnly: true 34 | securityContext: 35 | capabilities: 36 | add: 37 | - NET_ADMIN 38 | - SYS_TIME 39 | hostNetwork: true 40 | dnsPolicy: ClusterFirstWithHostNet 41 | volumes: 42 | - hostPath: 43 | path: /etc/kubernetes/ 44 | type: DirectoryOrCreate 45 | name: kubeconfig 46 | status: { } 47 | -------------------------------------------------------------------------------- /manifests/addons/local-storage.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: local-path-provisioner-service-account 6 | namespace: kube-system 7 | --- 8 | apiVersion: storage.k8s.io/v1 9 | kind: StorageClass 10 | metadata: 11 | name: local-path 12 | annotations: 13 | storageclass.kubernetes.io/is-default-class: "true" 14 | provisioner: rancher.io/local-path 15 | volumeBindingMode: WaitForFirstConsumer 16 | reclaimPolicy: Delete 17 | --- 18 | kind: ConfigMap 19 | apiVersion: v1 20 | metadata: 21 | name: local-path-config 22 | namespace: kube-system 23 | data: 24 | config.json: |- 25 | { 26 | "nodePathMap":[ 27 | { 28 | "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", 29 | "paths":["/mnt/storage"] 30 | } 31 | ] 32 | } 33 | --- 34 | apiVersion: rbac.authorization.k8s.io/v1 35 | kind: ClusterRole 36 | metadata: 37 | name: local-path-provisioner-role 38 | rules: 39 | - apiGroups: [ "" ] 40 | resources: [ "nodes", "persistentvolumeclaims" ] 41 | verbs: [ "get", "list", "watch" ] 42 | - apiGroups: [ "" ] 43 | resources: [ "endpoints", "persistentvolumes", "pods" ] 44 | verbs: [ "*" ] 45 | - apiGroups: [ "" ] 46 | resources: [ "events" ] 47 | verbs: [ "create", "patch" ] 48 | - apiGroups: [ "storage.k8s.io" ] 49 | resources: [ "storageclasses" ] 50 | verbs: [ "get", "list", "watch" ] 51 | --- 52 | apiVersion: rbac.authorization.k8s.io/v1 53 | kind: ClusterRoleBinding 54 | metadata: 55 | name: local-path-provisioner-bind 56 | roleRef: 57 | apiGroup: rbac.authorization.k8s.io 58 | kind: ClusterRole 59 | name: local-path-provisioner-role 60 | subjects: 61 | - kind: ServiceAccount 62 | name: local-path-provisioner-service-account 63 | namespace: kube-system 64 | --- 65 | apiVersion: apps/v1 66 | kind: Deployment 67 | metadata: 68 | name: local-path-provisioner 69 | namespace: kube-system 70 | spec: 71 | replicas: 1 72 | selector: 73 | matchLabels: 74 | app: local-path-provisioner 75 | template: 76 | metadata: 77 | labels: 78 | app: local-path-provisioner 79 | spec: 80 | serviceAccountName: local-path-provisioner-service-account 81 | tolerations: 82 | - key: "CriticalAddonsOnly" 83 | operator: "Exists" 84 | - key: "node-role.kubernetes.io/master" 85 | operator: "Exists" 86 | effect: "NoSchedule" 87 | containers: 88 | - name: local-path-provisioner 89 | image: rancher/local-path-provisioner:v0.0.14 90 | imagePullPolicy: IfNotPresent 91 | command: 92 | - local-path-provisioner 93 | - start 94 | - --config 95 | - /etc/config/config.json 96 | volumeMounts: 97 | - name: config-volume 98 | mountPath: /etc/config/ 99 | env: 100 | - name: POD_NAMESPACE 101 | valueFrom: 102 | fieldRef: 103 | fieldPath: metadata.namespace 104 | - name: HELPER_IMAGE 105 | value: rancher/library-busybox:1.31.1 106 | volumes: 107 | - name: config-volume 108 | configMap: 109 | name: local-path-config 110 | 111 | 112 | -------------------------------------------------------------------------------- /manifests/etcd-statefulset.yaml: -------------------------------------------------------------------------------- 1 | # k8s.gcr.io/etcd:3.4.13-0 2 | # docker.io/wtxue/etcd:3.4.13-0 3 | --- 4 | apiVersion: apps/v1 5 | kind: StatefulSet 6 | metadata: 7 | name: etcd 8 | labels: 9 | app: etcd 10 | version: blue 11 | spec: 12 | serviceName: etcd 13 | replicas: 3 14 | selector: 15 | matchLabels: 16 | app: etcd 17 | version: blue 18 | template: 19 | metadata: 20 | name: etcd 21 | labels: 22 | app: etcd 23 | version: blue 24 | spec: 25 | containers: 26 | - name: etcd 27 | image: docker.io/wtxue/etcd:3.4.13-0 28 | ports: 29 | - containerPort: 2379 30 | name: client 31 | - containerPort: 2380 32 | name: peer 33 | env: 34 | - name: INITIAL_CLUSTER_SIZE 35 | value: "3" 36 | - name: CLUSTER_NAMESPACE 37 | valueFrom: 38 | fieldRef: 39 | fieldPath: metadata.namespace 40 | # resources: 41 | # requests: 42 | # cpu: 100m 43 | # memory: 512Mi 44 | volumeMounts: 45 | - name: datadir 46 | mountPath: /var/run/etcd 47 | command: 48 | - /bin/sh 49 | - -c 50 | - | 51 | PEERS="etcd-0=http://etcd-0.etcd:2380,etcd-1=http://etcd-1.etcd:2380,etcd-2=http://etcd-2.etcd:2380" 52 | exec etcd --name ${HOSTNAME} \ 53 | --listen-peer-urls http://0.0.0.0:2380 \ 54 | --listen-client-urls http://0.0.0.0:2379 \ 55 | --advertise-client-urls http://${HOSTNAME}.etcd:2379 \ 56 | --initial-advertise-peer-urls http://${HOSTNAME}:2380 \ 57 | --initial-cluster-token etcd-cluster-1 \ 58 | --initial-cluster ${PEERS} \ 59 | --initial-cluster-state new \ 60 | --data-dir /var/run/etcd/default.etcd \ 61 | --logger=zap 62 | volumeClaimTemplates: 63 | - metadata: 64 | name: datadir 65 | spec: 66 | # storageClassName: local-storage 67 | accessModes: 68 | - "ReadWriteOnce" 69 | resources: 70 | requests: 71 | storage: 2Gi 72 | --- 73 | apiVersion: v1 74 | kind: Service 75 | metadata: 76 | name: etcd 77 | labels: 78 | app: etcd 79 | spec: 80 | ports: 81 | - port: 2380 82 | name: etcd-server 83 | - port: 2379 84 | name: etcd-client 85 | clusterIP: None 86 | selector: 87 | app: etcd 88 | publishNotReadyAddresses: true 89 | --- 90 | apiVersion: policy/v1beta1 91 | kind: PodDisruptionBudget 92 | metadata: 93 | name: etcd-pdb 94 | labels: 95 | pdb: etcd 96 | spec: 97 | minAvailable: 2 98 | selector: 99 | matchLabels: 100 | app: etcd 101 | --- 102 | -------------------------------------------------------------------------------- /manifests/ha-local-cluster-node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: devops.fake.io/v1 2 | kind: Machine 3 | metadata: 4 | labels: 5 | name: 172.16.18.20 6 | clusterName: ha-local-cluster 7 | name: 172.16.18.20 8 | namespace: ha-local-cluster 9 | spec: 10 | clusterName: ha-local-cluster 11 | type: baremetal 12 | machine: 13 | ip: 172.16.18.20 14 | port: 22 15 | username: root 16 | password: "123456" 17 | feature: 18 | hooks: 19 | installType: kubeadm 20 | -------------------------------------------------------------------------------- /manifests/ha-local-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | name: ha-local-cluster 6 | name: ha-local-cluster 7 | --- 8 | apiVersion: devops.fake.io/v1 9 | kind: Cluster 10 | metadata: 11 | name: ha-local-cluster 12 | namespace: ha-local-cluster 13 | annotations: 14 | # "fake.io/restore.step": "EnsurePreInstallHook" 15 | "fake.io/debug.localdir": "bin/linux/" 16 | "fake.io/update.step": EnsureDeployCni, EnsureMetricsServer 17 | # "fake.io/apiserver.vip": 172.16.18.17 18 | spec: 19 | pause: false 20 | tenantID: k8s 21 | displayName: demo 22 | clusterType: baremetal 23 | osType: ubuntu 24 | criType: containerd 25 | version: v1.20.3 26 | networkDevice: ens34 27 | clusterCIDR: 172.16.101.0/24 28 | serviceCIDR: 172.16.201.0/24 29 | dnsDomain: cluster.local 30 | publicAlternativeNames: 31 | - ha-local.vip.fake.io 32 | features: 33 | ipvs: true 34 | internalLB: true 35 | enableMasterSchedule: true 36 | ha: 37 | thirdParty: 38 | vip: "172.16.18.243" 39 | vport: 6443 40 | hooks: 41 | cniInstall: flannel 42 | properties: 43 | maxNodePodNum: 64 44 | machines: 45 | - ip: 172.16.18.17 46 | port: 22 47 | username: root 48 | password: "123456" 49 | - ip: 172.16.18.18 50 | port: 22 51 | username: root 52 | password: "123456" 53 | - ip: 172.16.18.19 54 | port: 22 55 | username: root 56 | password: "123456" 57 | apiServerExtraArgs: 58 | audit-log-maxage: "30" 59 | audit-log-maxbackup: "3" 60 | audit-log-maxsize: "100" 61 | audit-log-truncate-enabled: "true" 62 | audit-log-path: "/var/log/kubernetes/k8s-audit.log" 63 | controllerManagerExtraArgs: 64 | bind-address: "0.0.0.0" 65 | schedulerExtraArgs: 66 | bind-address: "0.0.0.0" 67 | -------------------------------------------------------------------------------- /manifests/managed-cluster-node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: devops.fake.io/v1 2 | kind: Machine 3 | metadata: 4 | labels: 5 | name: nc29pffvqtofq70ufrdb9 6 | clusterName: cc29pffvqtofq70ufrdb9 7 | name: nc29pffvqtofq70ufrdb9 8 | namespace: managed-cluster 9 | spec: 10 | clusterName: cc29pffvqtofq70ufrdb9 11 | type: managed 12 | machine: 13 | ip: 172.16.0.137 14 | port: 22 15 | username: root 16 | password: "xxxxxx" 17 | feature: 18 | hooks: 19 | installType: kubeadm 20 | -------------------------------------------------------------------------------- /manifests/managed-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | name: managed-cluster 6 | name: managed-cluster 7 | --- 8 | apiVersion: devops.fake.io/v1 9 | kind: Cluster 10 | metadata: 11 | name: cc29pffvqtofq70ufrdb9 12 | namespace: managed-cluster 13 | annotations: 14 | "fake.io/update.step": EnsureKubeconfig,EnsureAddons,EnsureCni,EnsureMetricsServer 15 | "fake.io/apiserver.vip": 172.16.0.157 16 | spec: 17 | pause: false 18 | tenantID: wtxue 19 | displayName: demo 20 | clusterType: managed 21 | osType: ubuntu 22 | criType: containerd 23 | version: v1.20.4 24 | networkType: eth0 25 | clusterCIDR: 10.97.0.0/16 26 | serviceCIDR: 10.96.0.0/16 27 | dnsDomain: cluster.local 28 | publicAlternativeNames: 29 | - cc29pffvqtofq70ufrdb9.fake.io 30 | features: 31 | ipvs: true 32 | internalLB: true 33 | enableMasterSchedule: true 34 | ha: 35 | thirdParty: 36 | vip: "172.16.0.157" 37 | vport: 6443 38 | files: 39 | - src: "/k8s/bin/k9s" 40 | dst: "/usr/local/bin/k9s" 41 | hooks: 42 | PostCniInstall: flannel 43 | properties: 44 | maxNodePodNum: 64 45 | apiServerExtraArgs: 46 | audit-log-maxage: "30" 47 | audit-log-maxbackup: "3" 48 | audit-log-maxsize: "100" 49 | audit-log-truncate-enabled: "true" 50 | audit-policy-file: "/etc/kubernetes/audit-policy.yaml" 51 | audit-log-path: "/var/log/kubernetes/k8s-audit.log" 52 | controllerManagerExtraArgs: 53 | "bind-address": "0.0.0.0" 54 | schedulerExtraArgs: 55 | "bind-address": "0.0.0.0" 56 | etcd: 57 | external: 58 | endpoints: 59 | - http://172.16.0.157:2379 60 | -------------------------------------------------------------------------------- /manifests/test/curl-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | name: nginx-deployment 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: nginx 9 | replicas: 10 10 | template: 11 | metadata: 12 | labels: 13 | app: nginx 14 | spec: 15 | containers: 16 | - name: nginx 17 | image: nginx:1.16.1 18 | ports: 19 | - containerPort: 80 20 | 21 | --- 22 | apiVersion: v1 23 | kind: Service 24 | metadata: 25 | name: nginx-deployment 26 | labels: 27 | kubernetes.io/name: "nginx-deployment" 28 | spec: 29 | selector: 30 | app: nginx 31 | ports: 32 | - port: 80 33 | protocol: TCP 34 | targetPort: 80 -------------------------------------------------------------------------------- /manifests/test/kuard.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: kuard 6 | name: kuard 7 | spec: 8 | replicas: 3 9 | selector: 10 | matchLabels: 11 | app: kuard 12 | template: 13 | metadata: 14 | labels: 15 | app: kuard 16 | spec: 17 | containers: 18 | - image: registry.cn-hangzhou.aliyuncs.com/wtxue/kuard-amd64:1 19 | name: kuard 20 | # gcr.io/kuar-demo/kuard-amd64:1 21 | # registry.cn-hangzhou.aliyuncs.com/wtxue/kuard-amd64:1 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | labels: 27 | app: kuard 28 | name: kuard 29 | spec: 30 | ports: 31 | - port: 80 32 | protocol: TCP 33 | targetPort: 8080 34 | selector: 35 | app: kuard 36 | sessionAffinity: None 37 | type: ClusterIP 38 | --- 39 | apiVersion: networking.k8s.io/v1beta1 40 | kind: Ingress 41 | metadata: 42 | name: kuard 43 | labels: 44 | app: kuard 45 | annotations: 46 | kubernetes.io/ingress.class: contour 47 | spec: 48 | rules: 49 | - host: kuard.k8s.io 50 | http: 51 | paths: 52 | - path: / 53 | backend: 54 | serviceName: kuard 55 | servicePort: 80 56 | 57 | 58 | -------------------------------------------------------------------------------- /manifests/test/local-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | name: local-cluster 6 | name: local-cluster 7 | --- 8 | apiVersion: devops.k8s.io/v1 9 | kind: Cluster 10 | metadata: 11 | name: local-cluster 12 | namespace: local-cluster 13 | annotations: 14 | # "k8s.io/step.restore": "EnsureCRI" 15 | "k8s.io/local.dir": "bin/linux/" 16 | "k8s.io/apply.step": EnsureDeployCni, EnsureMetricsServer 17 | spec: 18 | pause: false 19 | tenantID: k8s 20 | displayName: demo 21 | clusterType: baremetal 22 | osType: ubuntu 23 | criType: containerd 24 | version: v1.19.6 25 | networkDevice: ens34 26 | clusterCIDR: 172.16.99.0/24 27 | serviceCIDR: 172.16.100.0/24 28 | dnsDomain: cluster.local 29 | publicAlternativeNames: 30 | - vip.local.k8s.io 31 | features: 32 | ipvs: true 33 | internalLB: true 34 | enableMasterSchedule: true 35 | ha: 36 | thirdParty: 37 | vip: "172.16.18.241" 38 | vport: 6443 39 | hooks: 40 | cniInstall: flannel 41 | properties: 42 | maxNodePodNum: 64 43 | machines: 44 | - ip: 172.16.18.9 45 | port: 22 46 | username: root 47 | password: "123456" 48 | apiServerExtraArgs: 49 | audit-log-maxage: "30" 50 | audit-log-maxbackup: "3" 51 | audit-log-maxsize: "100" 52 | audit-log-truncate-enabled: "true" 53 | audit-log-path: "/var/log/kubernetes/k8s-audit.log" 54 | controllerManagerExtraArgs: 55 | bind-address: "0.0.0.0" 56 | schedulerExtraArgs: 57 | bind-address: "0.0.0.0" 58 | -------------------------------------------------------------------------------- /manifests/vcluster/readme.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | kubectl create namespace vcluster-namespace 4 | kubectl apply -f vcluster.yaml -n vcluster-namespace 5 | -------------------------------------------------------------------------------- /manifests/vcluster/vcluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: vcluster-1 5 | --- 6 | kind: Role 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | metadata: 9 | name: vcluster-1 10 | rules: 11 | - apiGroups: [""] 12 | resources: ["configmaps", "secrets", "services", "services/proxy", "pods", "pods/proxy", "pods/attach", "pods/portforward", "pods/exec", "pods/log", "events", "endpoints", "persistentvolumeclaims"] 13 | verbs: ["*"] 14 | - apiGroups: ["networking.k8s.io"] 15 | resources: ["ingresses"] 16 | verbs: ["*"] 17 | - apiGroups: [""] 18 | resources: ["namespaces"] 19 | verbs: ["get", "list", "watch"] 20 | - apiGroups: ["apps"] 21 | resources: ["statefulsets"] 22 | verbs: ["get", "list", "watch"] 23 | --- 24 | kind: RoleBinding 25 | apiVersion: rbac.authorization.k8s.io/v1 26 | metadata: 27 | name: vcluster-1 28 | subjects: 29 | - kind: ServiceAccount 30 | name: vcluster-1 31 | roleRef: 32 | kind: Role 33 | name: vcluster-1 34 | apiGroup: rbac.authorization.k8s.io 35 | --- 36 | apiVersion: v1 37 | kind: Service 38 | metadata: 39 | name: vcluster-1 40 | spec: 41 | type: ClusterIP 42 | ports: 43 | - name: https 44 | port: 443 45 | targetPort: 8443 46 | protocol: TCP 47 | selector: 48 | app: vcluster-1 49 | --- 50 | apiVersion: v1 51 | kind: Service 52 | metadata: 53 | name: vcluster-1-headless 54 | spec: 55 | ports: 56 | - name: https 57 | port: 443 58 | targetPort: 8443 59 | protocol: TCP 60 | clusterIP: None 61 | selector: 62 | app: vcluster-1 63 | --- 64 | apiVersion: apps/v1 65 | kind: StatefulSet 66 | metadata: 67 | name: vcluster-1 68 | labels: 69 | app: vcluster-1 70 | spec: 71 | serviceName: vcluster-1-headless 72 | replicas: 1 73 | selector: 74 | matchLabels: 75 | app: vcluster-1 76 | template: 77 | metadata: 78 | labels: 79 | app: vcluster-1 80 | spec: 81 | terminationGracePeriodSeconds: 10 82 | serviceAccountName: vcluster-1 83 | containers: 84 | - image: rancher/k3s:v1.19.5-k3s2 85 | name: virtual-cluster 86 | command: 87 | - "/bin/k3s" 88 | args: 89 | - "server" 90 | - "--write-kubeconfig=/k3s-config/kube-config.yaml" 91 | - "--data-dir=/data" 92 | - "--disable=traefik,servicelb,metrics-server,local-storage" 93 | - "--disable-network-policy" 94 | - "--disable-agent" 95 | - "--disable-scheduler" 96 | - "--disable-cloud-controller" 97 | - "--flannel-backend=none" 98 | - "--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle" 99 | - "--service-cidr=10.96.0.0/12" 100 | volumeMounts: 101 | - mountPath: /data 102 | name: data 103 | - name: syncer 104 | image: "loftsh/virtual-cluster:0.0.27" 105 | args: 106 | - --service-name=vcluster-1 107 | - --suffix=vcluster-1 108 | - --owning-statefulset=vcluster-1 109 | - --out-kube-config-secret=vcluster-1 110 | volumeMounts: 111 | - mountPath: /data 112 | name: data 113 | volumeClaimTemplates: 114 | - metadata: 115 | name: data 116 | spec: 117 | accessModes: [ "ReadWriteOnce" ] 118 | resources: 119 | requests: 120 | storage: 5Gi 121 | -------------------------------------------------------------------------------- /pkg/addons/kubevip/kubevip.go: -------------------------------------------------------------------------------- 1 | package kubevip 2 | 3 | import ( 4 | "github.com/wtxue/kok-operator/pkg/controllers/common" 5 | "github.com/wtxue/kok-operator/pkg/util/template" 6 | ) 7 | 8 | const ( 9 | kubeVipTemplate = ` 10 | apiVersion: v1 11 | kind: Pod 12 | metadata: 13 | creationTimestamp: null 14 | name: kube-vip 15 | namespace: kube-system 16 | spec: 17 | containers: 18 | - args: 19 | - start 20 | env: 21 | - name: vip_arp 22 | value: "true" 23 | - name: vip_interface 24 | value: "{{ default "eth0" .EthInterface }}" 25 | - name: vip_leaderelection 26 | value: "true" 27 | - name: vip_leaseduration 28 | value: "5" 29 | - name: vip_renewdeadline 30 | value: "3" 31 | - name: vip_retryperiod 32 | value: "1" 33 | - name: vip_address 34 | value: "{{ default "172.16.18.243" .Vip }}" 35 | image: "{{ default "plndr/kube-vip:0.2.3" .ImageName }}" 36 | imagePullPolicy: Always 37 | name: kube-vip 38 | resources: {} 39 | volumeMounts: 40 | - mountPath: /etc/kubernetes/ 41 | name: kubeconfig 42 | readOnly: true 43 | securityContext: 44 | capabilities: 45 | add: 46 | - NET_ADMIN 47 | - SYS_TIME 48 | hostNetwork: true 49 | dnsPolicy: ClusterFirstWithHostNet 50 | volumes: 51 | - hostPath: 52 | path: /etc/kubernetes/ 53 | type: DirectoryOrCreate 54 | name: kubeconfig 55 | status: {} 56 | ` 57 | ) 58 | 59 | type Option struct { 60 | EthInterface string 61 | Vip string 62 | ImageName string 63 | } 64 | 65 | func BuildKubeVipStaticPod(ctx *common.ClusterContext) map[string]string { 66 | vip := "" 67 | if ctx.Cluster.Spec.Features.HA != nil { 68 | if ctx.Cluster.Spec.Features.HA.KubeHA != nil { 69 | vip = ctx.Cluster.Spec.Features.HA.KubeHA.VIP 70 | } 71 | if ctx.Cluster.Spec.Features.HA.ThirdPartyHA != nil { 72 | vip = ctx.Cluster.Spec.Features.HA.ThirdPartyHA.VIP 73 | } 74 | } 75 | 76 | opt := &Option{ 77 | EthInterface: ctx.Cluster.Spec.NetworkDevice, 78 | Vip: vip, 79 | ImageName: "", 80 | } 81 | 82 | data, err := template.ParseString(kubeVipTemplate, opt) 83 | if err != nil { 84 | return nil 85 | } 86 | 87 | staticPodMap := map[string]string{ 88 | "kube-vip.yaml": string(data), 89 | } 90 | 91 | return staticPodMap 92 | } 93 | -------------------------------------------------------------------------------- /pkg/addons/metallb/doc.go: -------------------------------------------------------------------------------- 1 | package metallb 2 | -------------------------------------------------------------------------------- /pkg/apis/devops/v1/clusterCredential_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 4 | 5 | type CredentialInfo struct { 6 | TenantID string `json:"tenantID"` 7 | ClusterName string `json:"clusterName"` 8 | 9 | // For TKE in global reuse 10 | // +optional 11 | ETCDCACert []byte `json:"etcdCACert,omitempty"` 12 | // +optional 13 | ETCDCAKey []byte `json:"etcdCAKey,omitempty"` 14 | // +optional 15 | ETCDAPIClientCert []byte `json:"etcdAPIClientCert,omitempty"` 16 | // +optional 17 | ETCDAPIClientKey []byte `json:"etcdAPIClientKey,omitempty"` 18 | 19 | // For connect the cluster 20 | // +optional 21 | CACert []byte `json:"caCert,omitempty"` 22 | // +optional 23 | CAKey []byte `json:"caKey,omitempty"` 24 | // For kube-apiserver X509 auth 25 | // +optional 26 | ClientCert []byte `json:"clientCert,omitempty"` 27 | // For kube-apiserver X509 auth 28 | // +optional 29 | ClientKey []byte `json:"clientKey,omitempty"` 30 | // For kube-apiserver token auth 31 | // +optional 32 | Token *string `json:"token,omitempty"` 33 | // For kubeadm init or join 34 | // +optional 35 | BootstrapToken *string `json:"bootstrapToken,omitempty"` 36 | // For kubeadm init or join 37 | // +optional 38 | CertificateKey *string `json:"certificateKey,omitempty"` 39 | 40 | ExtData map[string]string `json:"extData,omitempty"` 41 | KubeData map[string]string `json:"kubeData,omitempty"` 42 | ManifestsData map[string]string `json:"manifestsData,omitempty"` 43 | CertsBinaryData map[string][]byte `json:"certsBinaryData,omitempty"` 44 | } 45 | 46 | // +kubebuilder:object:root=true 47 | 48 | // ClusterCredential records the credential information needed to access the cluster. 49 | type ClusterCredential struct { 50 | metav1.TypeMeta `json:",inline"` 51 | // +optional 52 | metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 53 | 54 | CredentialInfo `json:",inline"` 55 | } 56 | 57 | // +kubebuilder:object:root=true 58 | 59 | // ClusterCredentialList is the whole list of all ClusterCredential which owned by a tenant. 60 | type ClusterCredentialList struct { 61 | metav1.TypeMeta `json:",inline"` 62 | // +optional 63 | metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 64 | 65 | // List of clusters 66 | Items []ClusterCredential `json:"items" protobuf:"bytes,2,rep,name=items"` 67 | } 68 | -------------------------------------------------------------------------------- /pkg/apis/devops/v1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Package v1 contains API Schema definitions for the devops v1 API group 2 | // +kubebuilder:object:generate=true 3 | // +groupName=devops.fake.io 4 | package v1 5 | 6 | import ( 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "sigs.k8s.io/controller-runtime/pkg/scheme" 9 | ) 10 | 11 | var ( 12 | // GroupVersion is group version used to register these objects 13 | GroupVersion = schema.GroupVersion{Group: "devops.fake.io", Version: "v1"} 14 | 15 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 16 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 17 | 18 | // AddToScheme adds the types in this group-version to the given scheme. 19 | AddToScheme = SchemeBuilder.AddToScheme 20 | ) 21 | 22 | func init() { 23 | SchemeBuilder.Register(&Cluster{}, &ClusterList{}) 24 | SchemeBuilder.Register(&Machine{}, &MachineList{}) 25 | SchemeBuilder.Register(&ClusterCredential{}, &ClusterCredentialList{}) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/apis/helper.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/apimachinery/pkg/runtime/schema" 7 | "k8s.io/apimachinery/pkg/runtime/serializer" 8 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 9 | kubeproxyv1alpha1 "k8s.io/kube-proxy/config/v1alpha1" 10 | kubeletv1beta1 "k8s.io/kubelet/config/v1beta1" 11 | kubeadmv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" 12 | kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" 13 | ) 14 | 15 | var Scheme = runtime.NewScheme() 16 | 17 | var Codecs = serializer.NewCodecFactory(Scheme) 18 | 19 | var localSchemeBuilder = runtime.SchemeBuilder{ 20 | kubeadmv1beta2.AddToScheme, 21 | kubeadmv1beta3.AddToScheme, 22 | kubeletv1beta1.AddToScheme, 23 | kubeproxyv1alpha1.AddToScheme, 24 | } 25 | 26 | var AddToScheme = localSchemeBuilder.AddToScheme 27 | 28 | func init() { 29 | utilruntime.Must(AddToScheme(Scheme)) 30 | } 31 | 32 | // MarshalToYAML marshals an object into yaml. 33 | func MarshalToYAML(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) { 34 | const mediaType = runtime.ContentTypeYAML 35 | info, ok := runtime.SerializerInfoForMediaType(Codecs.SupportedMediaTypes(), mediaType) 36 | if !ok { 37 | return []byte{}, errors.Errorf("unsupported media type %q", mediaType) 38 | } 39 | encoder := Codecs.EncoderForVersion(info.Serializer, gv) 40 | return runtime.Encode(encoder, obj) 41 | } 42 | 43 | // GetScheme gets an initialized runtime.Scheme with kubeadm, kubelet, kubeproxy 44 | func GetScheme() *runtime.Scheme { 45 | return Scheme 46 | } 47 | -------------------------------------------------------------------------------- /pkg/apis/types.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" 5 | ) 6 | 7 | // WarpperConfiguration contains a list of elements that InitConfiguration and ClusterConfiguration 8 | type WarpperConfiguration struct { 9 | *kubeadmv1beta3.InitConfiguration `json:"-"` 10 | *kubeadmv1beta3.ClusterConfiguration `json:"-"` 11 | IPs []string 12 | } 13 | -------------------------------------------------------------------------------- /pkg/apis/workload/v1/addons_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // AddonsSpec is a description of Addons. 8 | type AddonsSpec struct { 9 | Foo string `json:"foo,omitempty"` 10 | } 11 | 12 | // AddonsStatus represents information about the status of an Addons. 13 | type AddonsStatus struct { 14 | Phase string `json:"phase,omitempty"` 15 | Foo string `json:"foo,omitempty"` 16 | } 17 | 18 | // +genclient 19 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 20 | 21 | // +kubebuilder:object:root=true 22 | 23 | // Addons is the Schema for the Addon API 24 | // +k8s:openapi-gen=true 25 | // +kubebuilder:subresource:status 26 | // +kubebuilder:printcolumn:name="PHASE",type="string",JSONPath=".status.phase",description="The Addons phase." 27 | // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. " 28 | type Addons struct { 29 | metav1.TypeMeta `json:",inline"` 30 | metav1.ObjectMeta `json:"metadata,omitempty"` 31 | 32 | Spec AddonsSpec `json:"spec,omitempty"` 33 | Status AddonsStatus `json:"status,omitempty"` 34 | } 35 | 36 | // +kubebuilder:object:root=true 37 | 38 | // AddonsList contains a list of Addons 39 | type AddonsList struct { 40 | metav1.TypeMeta `json:",inline"` 41 | metav1.ListMeta `json:"metadata,omitempty"` 42 | Items []Addons `json:"items"` 43 | } 44 | -------------------------------------------------------------------------------- /pkg/apis/workload/v1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Package v1 contains API Schema definitions for the app v1 API group 2 | // +kubebuilder:object:generate=true 3 | // +groupName=workload.fake.io 4 | package v1 5 | 6 | import ( 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "sigs.k8s.io/controller-runtime/pkg/scheme" 9 | ) 10 | 11 | var ( 12 | // GroupVersion is group version used to register these objects 13 | GroupVersion = schema.GroupVersion{Group: "workload.fake.io", Version: "v1"} 14 | 15 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 16 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 17 | 18 | // AddToScheme adds the types in this group-version to the given scheme. 19 | AddToScheme = SchemeBuilder.AddToScheme 20 | ) 21 | 22 | func init() { 23 | SchemeBuilder.Register(&Addons{}, &AddonsList{}) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/apis/workload/v1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright 2021 wtxue. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by controller-gen. DO NOT EDIT. 20 | 21 | package v1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *Addons) DeepCopyInto(out *Addons) { 29 | *out = *in 30 | out.TypeMeta = in.TypeMeta 31 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 32 | out.Spec = in.Spec 33 | out.Status = in.Status 34 | } 35 | 36 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Addons. 37 | func (in *Addons) DeepCopy() *Addons { 38 | if in == nil { 39 | return nil 40 | } 41 | out := new(Addons) 42 | in.DeepCopyInto(out) 43 | return out 44 | } 45 | 46 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 47 | func (in *Addons) DeepCopyObject() runtime.Object { 48 | if c := in.DeepCopy(); c != nil { 49 | return c 50 | } 51 | return nil 52 | } 53 | 54 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 55 | func (in *AddonsList) DeepCopyInto(out *AddonsList) { 56 | *out = *in 57 | out.TypeMeta = in.TypeMeta 58 | in.ListMeta.DeepCopyInto(&out.ListMeta) 59 | if in.Items != nil { 60 | in, out := &in.Items, &out.Items 61 | *out = make([]Addons, len(*in)) 62 | for i := range *in { 63 | (*in)[i].DeepCopyInto(&(*out)[i]) 64 | } 65 | } 66 | } 67 | 68 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsList. 69 | func (in *AddonsList) DeepCopy() *AddonsList { 70 | if in == nil { 71 | return nil 72 | } 73 | out := new(AddonsList) 74 | in.DeepCopyInto(out) 75 | return out 76 | } 77 | 78 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 79 | func (in *AddonsList) DeepCopyObject() runtime.Object { 80 | if c := in.DeepCopy(); c != nil { 81 | return c 82 | } 83 | return nil 84 | } 85 | 86 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 87 | func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) { 88 | *out = *in 89 | } 90 | 91 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec. 92 | func (in *AddonsSpec) DeepCopy() *AddonsSpec { 93 | if in == nil { 94 | return nil 95 | } 96 | out := new(AddonsSpec) 97 | in.DeepCopyInto(out) 98 | return out 99 | } 100 | 101 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 102 | func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) { 103 | *out = *in 104 | } 105 | 106 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus. 107 | func (in *AddonsStatus) DeepCopy() *AddonsStatus { 108 | if in == nil { 109 | return nil 110 | } 111 | out := new(AddonsStatus) 112 | in.DeepCopyInto(out) 113 | return out 114 | } 115 | -------------------------------------------------------------------------------- /pkg/apiserver/internal/control_plane.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 wtxue. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package internal 18 | 19 | import ( 20 | "net/url" 21 | "text/template" 22 | 23 | "bytes" 24 | 25 | "k8s.io/apimachinery/pkg/runtime/serializer" 26 | "k8s.io/client-go/kubernetes/scheme" 27 | "k8s.io/client-go/rest" 28 | ) 29 | 30 | // ControlPlane is a struct that knows how to start your test control plane. 31 | // 32 | // Right now, that means Etcd and your APIServer. This is likely to increase in 33 | // future. 34 | type ControlPlane struct { 35 | APIServer *APIServer 36 | Etcd *Etcd 37 | } 38 | 39 | // Start will start your control plane processes. To stop them, call Stop(). 40 | func (f *ControlPlane) Start() error { 41 | if f.Etcd == nil { 42 | f.Etcd = &Etcd{} 43 | } 44 | if err := f.Etcd.Start(); err != nil { 45 | return err 46 | } 47 | 48 | if f.APIServer == nil { 49 | f.APIServer = &APIServer{} 50 | } 51 | f.APIServer.EtcdURL = f.Etcd.URL 52 | return f.APIServer.Start() 53 | } 54 | 55 | // Stop will stop your control plane processes, and clean up their data. 56 | func (f *ControlPlane) Stop() error { 57 | if f.APIServer != nil { 58 | if err := f.APIServer.Stop(); err != nil { 59 | return err 60 | } 61 | } 62 | if f.Etcd != nil { 63 | if err := f.Etcd.Stop(); err != nil { 64 | return err 65 | } 66 | } 67 | return nil 68 | } 69 | 70 | // APIURL returns the URL you should connect to to talk to your API. 71 | func (f *ControlPlane) APIURL() *url.URL { 72 | return f.APIServer.URL 73 | } 74 | 75 | // RESTClientConfig returns a pre-configured restconfig, ready to connect to 76 | // this ControlPlane. 77 | func (f *ControlPlane) RESTClientConfig() (*rest.Config, error) { 78 | c := &rest.Config{ 79 | Host: f.APIURL().String(), 80 | ContentConfig: rest.ContentConfig{ 81 | NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs}, 82 | }, 83 | } 84 | err := rest.SetKubernetesDefaults(c) 85 | return c, err 86 | } 87 | 88 | // RenderTemplates returns an []string to render the templates 89 | func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) { 90 | var t *template.Template 91 | 92 | for _, arg := range argTemplates { 93 | t, err = template.New(arg).Parse(arg) 94 | if err != nil { 95 | args = nil 96 | return 97 | } 98 | 99 | buf := &bytes.Buffer{} 100 | err = t.Execute(buf, data) 101 | if err != nil { 102 | args = nil 103 | return 104 | } 105 | args = append(args, buf.String()) 106 | } 107 | 108 | return 109 | } 110 | -------------------------------------------------------------------------------- /pkg/clustermanager/cluster.go: -------------------------------------------------------------------------------- 1 | package clustermanager 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/go-logr/logr" 10 | "github.com/pkg/errors" 11 | "github.com/wtxue/kok-operator/pkg/k8sclient" 12 | "k8s.io/apimachinery/pkg/util/runtime" 13 | "k8s.io/client-go/kubernetes" 14 | "sigs.k8s.io/controller-runtime/pkg/cluster" 15 | ) 16 | 17 | type ClusterStatusType string 18 | 19 | // These are valid status of a cluster. 20 | const ( 21 | // ClusterReady means the cluster is ready to accept workloads. 22 | ClusterReady ClusterStatusType = "Ready" 23 | // ClusterOffline means the cluster is temporarily down or not reachable 24 | ClusterOffline ClusterStatusType = "Offline" 25 | ClusterMaintain ClusterStatusType = "Maintaining" 26 | ) 27 | 28 | var ( 29 | SyncPeriodTime = 1 * time.Hour 30 | ) 31 | 32 | type Cluster struct { 33 | Name string 34 | AliasName string 35 | RawKubeconfig []byte 36 | Meta map[string]string 37 | KubeCli kubernetes.Interface 38 | 39 | cluster.Cluster 40 | SyncPeriod time.Duration 41 | Log logr.Logger 42 | 43 | StopperCancel context.CancelFunc 44 | 45 | Status ClusterStatusType 46 | // Started is true if the Informers has been Started 47 | Started bool 48 | } 49 | 50 | func NewCluster(name string, kubeconfig []byte, log logr.Logger) (*Cluster, error) { 51 | c := &Cluster{ 52 | Name: name, 53 | RawKubeconfig: kubeconfig, 54 | Log: log.WithValues("cluster", name), 55 | SyncPeriod: SyncPeriodTime, 56 | Started: false, 57 | } 58 | 59 | err := c.initK8SClients() 60 | if err != nil { 61 | return nil, errors.Wrapf(err, "could not re-init k8s clients name:%s", name) 62 | } 63 | 64 | return c, nil 65 | } 66 | 67 | func (c *Cluster) GetName() string { 68 | return c.Name 69 | } 70 | 71 | func (c *Cluster) initK8SClients() error { 72 | startTime := time.Now() 73 | cfg, err := k8sclient.NewClientConfig(c.RawKubeconfig) 74 | if err != nil { 75 | return errors.Wrapf(err, "could not get rest config name: %s", c.Name) 76 | } 77 | 78 | kubecli, err := kubernetes.NewForConfig(cfg) 79 | if err != nil { 80 | return errors.Wrapf(err, "could not new kubecli name:%s", c.Name) 81 | } 82 | 83 | c.Log.Info("new kube cli", "time taken", fmt.Sprintf("%v", time.Since(startTime))) 84 | cs, err := cluster.New(cfg, func(o *cluster.Options) { 85 | o.Scheme = k8sclient.GetScheme() 86 | o.SyncPeriod = &c.SyncPeriod 87 | }) 88 | 89 | c.Cluster = cs 90 | c.KubeCli = kubecli 91 | c.Log.Info("new kube manager", "time taken", fmt.Sprintf("%v", time.Since(startTime))) 92 | return nil 93 | } 94 | 95 | func (c *Cluster) healthCheck() bool { 96 | body, err := c.KubeCli.Discovery().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).Raw() 97 | if err != nil { 98 | runtime.HandleError(errors.Wrapf(err, "Failed to do cluster health check for cluster %q", c.Name)) 99 | c.Status = ClusterOffline 100 | return false 101 | } 102 | 103 | if !strings.EqualFold(string(body), "ok") { 104 | c.Status = ClusterOffline 105 | return false 106 | } 107 | c.Status = ClusterReady 108 | return true 109 | } 110 | 111 | func (c *Cluster) StartCache(stopCtx context.Context) { 112 | if c.Started { 113 | c.Log.Info("cache informers is already startd") 114 | return 115 | } 116 | 117 | c.Log.Info("start cache informers ... ") 118 | ctx, cancelFunc := context.WithCancel(context.Background()) 119 | go func() { 120 | err := c.GetCache().Start(ctx) 121 | if err != nil { 122 | c.Log.Error(err, "cache Informers quit") 123 | } 124 | }() 125 | 126 | c.GetCache().WaitForCacheSync(stopCtx) 127 | c.Started = true 128 | c.StopperCancel = cancelFunc 129 | } 130 | 131 | func (c *Cluster) Stop() { 132 | c.Log.Info("start stop cache informers") 133 | c.StopperCancel() 134 | } 135 | -------------------------------------------------------------------------------- /pkg/constants/finalizers.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | FinalizersCluster = "finalizers.fake.io/cluster" 5 | FinalizersMachine = "finalizers.fake.io/machine" 6 | ) 7 | 8 | func ContainsString(slice []string, s string) bool { 9 | for _, item := range slice { 10 | if item == s { 11 | return true 12 | } 13 | } 14 | return false 15 | } 16 | 17 | func RemoveString(slice []string, s string) (result []string) { 18 | for _, item := range slice { 19 | if item == s { 20 | continue 21 | } 22 | result = append(result, item) 23 | } 24 | return 25 | } 26 | -------------------------------------------------------------------------------- /pkg/constants/labels.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | const ( 8 | ComponentNameCrd = "crds" 9 | CreatedByLabel = "fake.io/created-by" 10 | CreatedBy = "operator" 11 | 12 | KubeApiServer = "kube-apiserver" 13 | KubeKubeScheduler = "kube-scheduler" 14 | KubeControllerManager = "kube-controller-manager" 15 | KubeApiServerCerts = "kube-apiserver-certs" 16 | KubeApiServerConfig = "kube-apiserver-config" 17 | KubeApiServerAudit = "kube-apiserver-audit" 18 | ) 19 | 20 | const ( 21 | ClusterUpdateStep = "fake.io/update.step" 22 | ClusterRestoreStep = "fake.io/restore.step" 23 | ClusterApiserverType = "fake.io/apiserver.type" 24 | ClusterApiserverVip = "fake.io/apiserver.vip" 25 | ClusterDebugLocalDir = "fake.io/debug.localdir" 26 | ) 27 | 28 | var CtrlLabels = map[string]string{ 29 | "createBy": "controller", 30 | } 31 | 32 | func GetMapKey(annotation map[string]string, key string) string { 33 | if k, ok := annotation[key]; ok { 34 | return k 35 | } 36 | 37 | return "" 38 | } 39 | 40 | // GenComponentName ... 41 | func GenComponentName(clusterID, suffix string) string { 42 | return fmt.Sprintf("%s-%s", clusterID, suffix) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/constants/misc.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | ManagedNodeLabelsKey = "fake.io/managed-apiserver" 5 | 6 | KubeadmConfigFileName = KubernetesDir + "kubeadm-config.yaml" 7 | 8 | KubeletService = ` 9 | [Unit] 10 | Description=kubelet: The Kubernetes Node Agent 11 | Documentation=https://kubernetes.io/docs/ 12 | 13 | [Service] 14 | User=root 15 | ExecStart=/usr/bin/kubelet 16 | Restart=always 17 | StartLimitInterval=0 18 | RestartSec=10 19 | 20 | [Install] 21 | WantedBy=multi-user.target 22 | ` 23 | 24 | KubeletServiceRunConfig = ` 25 | [Service] 26 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" 27 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 28 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 29 | EnvironmentFile=-/etc/sysconfig/kubelet 30 | ExecStart= 31 | ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS 32 | ` 33 | ) 34 | -------------------------------------------------------------------------------- /pkg/constants/ports.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | // ProxyStatusPort is the default port for the proxy metrics server. 5 | // May be overridden by a flag at startup. 6 | ProxyStatusPort = 10249 7 | // KubeletPort is the default port for the kubelet server on each host machine. 8 | // May be overridden by a flag at startup. 9 | KubeletPort = 10250 10 | // InsecureSchedulerPort is the default port for the scheduler status server. 11 | // May be overridden by a flag at startup. 12 | // Deprecated: use the secure KubeSchedulerPort instead. 13 | InsecureSchedulerPort = 10251 14 | // InsecureKubeControllerManagerPort is the default port for the controller manager status server. 15 | // May be overridden by a flag at startup. 16 | // Deprecated: use the secure KubeControllerManagerPort instead. 17 | InsecureKubeControllerManagerPort = 10252 18 | // InsecureCloudControllerManagerPort is the default port for the cloud controller manager server. 19 | // This value may be overridden by a flag at startup. 20 | // Deprecated: use the secure CloudControllerManagerPort instead. 21 | InsecureCloudControllerManagerPort = 10253 22 | // KubeletReadOnlyPort exposes basic read-only services from the kubelet. 23 | // May be overridden by a flag at startup. 24 | // This is necessary for heapster to collect monitoring stats from the kubelet 25 | // until heapster can transition to using the SSL endpoint. 26 | // TODO(roberthbailey): Remove this once we have a better solution for heapster. 27 | KubeletReadOnlyPort = 10255 28 | // ProxyHealthzPort is the default port for the proxy healthz server. 29 | // May be overridden by a flag at startup. 30 | ProxyHealthzPort = 10256 31 | // KubeControllerManagerPort is the default port for the controller manager status server. 32 | // May be overridden by a flag at startup. 33 | KubeControllerManagerPort = 10257 34 | // CloudControllerManagerPort is the default port for the cloud controller manager server. 35 | // This value may be overridden by a flag at startup. 36 | CloudControllerManagerPort = 10258 37 | 38 | // KubeSchedulerPort is the default port for the scheduler status server. 39 | // May be overridden by a flag at startup. 40 | KubeSchedulerPort = 10259 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/constants/version.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | var ( 4 | OSs = []string{"linux"} 5 | K8sVersions = []string{"v1.24.4"} 6 | CNIPluginsVersions = []string{"v1.1.1"} 7 | ContainerdVersions = []string{"1.6.8"} 8 | ) 9 | -------------------------------------------------------------------------------- /pkg/controllers/addons/addons.go: -------------------------------------------------------------------------------- 1 | package addons 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-logr/logr" 7 | workloadv1 "github.com/wtxue/kok-operator/pkg/apis/workload/v1" 8 | "github.com/wtxue/kok-operator/pkg/gmanager" 9 | apierrors "k8s.io/apimachinery/pkg/api/errors" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | ctrl "sigs.k8s.io/controller-runtime" 12 | "sigs.k8s.io/controller-runtime/pkg/client" 13 | logf "sigs.k8s.io/controller-runtime/pkg/log" 14 | "sigs.k8s.io/controller-runtime/pkg/manager" 15 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 16 | ) 17 | 18 | const ( 19 | controllerName = "addons" 20 | ) 21 | 22 | // addonsReconciler reconciles a addons object 23 | type addonsReconciler struct { 24 | client.Client 25 | Log logr.Logger 26 | Mgr manager.Manager 27 | Scheme *runtime.Scheme 28 | } 29 | 30 | type addonsContext struct { 31 | Ctx context.Context 32 | Req reconcile.Request 33 | Addons *workloadv1.Addons 34 | logr.Logger 35 | } 36 | 37 | func Add(mgr manager.Manager, pMgr *gmanager.GManager) error { 38 | r := &addonsReconciler{ 39 | Client: mgr.GetClient(), 40 | Mgr: mgr, 41 | Log: logf.Log.WithName(controllerName), 42 | Scheme: mgr.GetScheme(), 43 | } 44 | 45 | return ctrl.NewControllerManagedBy(mgr). 46 | For(&workloadv1.Addons{}). 47 | Complete(r) 48 | } 49 | 50 | func (r *addonsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { 51 | logger := r.Log.WithValues(controllerName, req.NamespacedName.String()) 52 | 53 | addons := &workloadv1.Addons{} 54 | err := r.Client.Get(ctx, req.NamespacedName, addons) 55 | if err != nil { 56 | if apierrors.IsNotFound(err) { 57 | logger.Error(err, "not find addons") 58 | return reconcile.Result{}, nil 59 | } 60 | 61 | logger.Error(err, "failed to get addons") 62 | return reconcile.Result{}, err 63 | } 64 | 65 | return r.reconcile(&addonsContext{Ctx: ctx, Req: req, Addons: addons, Logger: logger}) 66 | } 67 | 68 | func (r *addonsReconciler) reconcile(ctx *addonsContext) (reconcile.Result, error) { 69 | 70 | return reconcile.Result{}, nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/controllers/cluster/cluster_helper.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "time" 5 | 6 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 7 | "github.com/wtxue/kok-operator/pkg/controllers/common" 8 | "github.com/wtxue/kok-operator/pkg/provider/cluster" 9 | "k8s.io/apimachinery/pkg/api/equality" 10 | apierrors "k8s.io/apimachinery/pkg/api/errors" 11 | "k8s.io/apimachinery/pkg/api/meta" 12 | ) 13 | 14 | const ( 15 | clusterClientRetryCount = 5 16 | clusterClientRetryInterval = 5 * time.Second 17 | 18 | reasonFailedInit = "FailedInit" 19 | reasonFailedUpdate = "FailedUpdate" 20 | ) 21 | 22 | func (r *clusterReconciler) applyStatus(ctx *common.ClusterContext) error { 23 | credential := &devopsv1.ClusterCredential{} 24 | err := r.Client.Get(ctx.Ctx, ctx.Key, credential) 25 | if err != nil { 26 | if apierrors.IsNotFound(err) { 27 | ctx.Error(err, "not find cluster credential") 28 | return nil 29 | } 30 | 31 | ctx.Error(err, "failed to get cluster credential") 32 | return err 33 | } 34 | 35 | if !equality.Semantic.DeepEqual(credential.CredentialInfo, ctx.Credential.CredentialInfo) { 36 | metaAccessor := meta.NewAccessor() 37 | currentResourceVersion, err := metaAccessor.ResourceVersion(credential) 38 | if err != nil { 39 | ctx.Error(err, "failed to metaAccessor") 40 | return err 41 | } 42 | metaAccessor.SetResourceVersion(ctx.Credential, currentResourceVersion) 43 | err = r.Client.Update(ctx.Ctx, ctx.Credential) 44 | if err != nil { 45 | ctx.Error(err, "failed to update cluster credential") 46 | return err 47 | } 48 | ctx.V(4).Info("update cluster credential success") 49 | } 50 | 51 | c := &devopsv1.Cluster{} 52 | err = r.Client.Get(ctx.Ctx, ctx.Key, c) 53 | if err != nil { 54 | if apierrors.IsNotFound(err) { 55 | ctx.Error(err, "not find cluster") 56 | return nil 57 | } 58 | 59 | ctx.Error(err, "failed to get cluster") 60 | return err 61 | } 62 | 63 | if !equality.Semantic.DeepEqual(c.Status, ctx.Cluster.Status) { 64 | metaAccessor := meta.NewAccessor() 65 | currentResourceVersion, err := metaAccessor.ResourceVersion(c) 66 | if err != nil { 67 | ctx.Error(err, "failed to metaAccessor") 68 | return err 69 | } 70 | 71 | metaAccessor.SetResourceVersion(ctx.Cluster, currentResourceVersion) 72 | err = r.Client.Status().Update(ctx.Ctx, ctx.Cluster) 73 | if err != nil { 74 | ctx.Error(err, "failed to update cluster status") 75 | return err 76 | } 77 | 78 | ctx.Info("update cluster status success") 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func (r *clusterReconciler) onCreate(ctx *common.ClusterContext, p cluster.Provider) error { 85 | err := p.OnCreate(ctx) 86 | if err != nil { 87 | ctx.Cluster.Status.Message = err.Error() 88 | ctx.Cluster.Status.Reason = reasonFailedInit 89 | } else { 90 | condition := ctx.Cluster.Status.Conditions[len(ctx.Cluster.Status.Conditions)-1] 91 | if condition.Status == devopsv1.ConditionFalse { // means current condition run into error 92 | ctx.Cluster.Status.Message = condition.Message 93 | ctx.Cluster.Status.Reason = condition.Reason 94 | } else { 95 | ctx.Cluster.Status.Message = "" 96 | ctx.Cluster.Status.Reason = "" 97 | } 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func (r *clusterReconciler) onUpdate(ctx *common.ClusterContext, p cluster.Provider) error { 104 | err := p.OnUpdate(ctx) 105 | if err != nil { 106 | ctx.Cluster.Status.Message = err.Error() 107 | ctx.Cluster.Status.Reason = reasonFailedUpdate 108 | } else { 109 | ctx.Cluster.Status.Message = "" 110 | ctx.Cluster.Status.Reason = "" 111 | } 112 | 113 | return nil 114 | } 115 | -------------------------------------------------------------------------------- /pkg/controllers/common/templates.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/wtxue/kok-operator/pkg/k8sutil" 5 | "k8s.io/apimachinery/pkg/api/meta" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | ) 9 | 10 | func ObjectMeta(name string, labels map[string]string, config runtime.Object) metav1.ObjectMeta { 11 | obj := config.DeepCopyObject() 12 | objMeta, _ := meta.Accessor(obj) 13 | ovk := config.GetObjectKind().GroupVersionKind() 14 | 15 | return metav1.ObjectMeta{ 16 | Name: name, 17 | Namespace: objMeta.GetNamespace(), 18 | Labels: labels, 19 | OwnerReferences: []metav1.OwnerReference{ 20 | { 21 | APIVersion: ovk.GroupVersion().String(), 22 | Kind: ovk.Kind, 23 | Name: objMeta.GetName(), 24 | UID: objMeta.GetUID(), 25 | Controller: k8sutil.BoolPointer(true), 26 | BlockOwnerDeletion: k8sutil.BoolPointer(true), 27 | }, 28 | }, 29 | } 30 | } 31 | 32 | func ObjectMetaWithAnnotations(name string, labels map[string]string, annotations map[string]string, config runtime.Object) metav1.ObjectMeta { 33 | o := ObjectMeta(name, labels, config) 34 | o.Annotations = annotations 35 | return o 36 | } 37 | 38 | func ObjectMetaClusterScope(name string, labels map[string]string, config runtime.Object) metav1.ObjectMeta { 39 | obj := config.DeepCopyObject() 40 | objMeta, _ := meta.Accessor(obj) 41 | ovk := config.GetObjectKind().GroupVersionKind() 42 | 43 | return metav1.ObjectMeta{ 44 | Name: name, 45 | Labels: labels, 46 | OwnerReferences: []metav1.OwnerReference{ 47 | { 48 | APIVersion: ovk.GroupVersion().String(), 49 | Kind: ovk.Kind, 50 | Name: objMeta.GetName(), 51 | UID: objMeta.GetUID(), 52 | Controller: k8sutil.BoolPointer(true), 53 | BlockOwnerDeletion: k8sutil.BoolPointer(true), 54 | }, 55 | }, 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/controllers/controller.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "github.com/wtxue/kok-operator/pkg/clustermanager" 5 | "github.com/wtxue/kok-operator/pkg/controllers/addons" 6 | "github.com/wtxue/kok-operator/pkg/controllers/cluster" 7 | "github.com/wtxue/kok-operator/pkg/controllers/machine" 8 | "github.com/wtxue/kok-operator/pkg/gmanager" 9 | "github.com/wtxue/kok-operator/pkg/option" 10 | "github.com/wtxue/kok-operator/pkg/provider" 11 | "github.com/wtxue/kok-operator/pkg/provider/config" 12 | "sigs.k8s.io/controller-runtime/pkg/manager" 13 | ) 14 | 15 | // AddToManagerFuncs is a list of functions to add all Controllers to the Manager 16 | var AddToManagerFuncs []func(manager.Manager) error 17 | 18 | var AddToManagerWithProviderFuncs []func(manager.Manager, *gmanager.GManager) error 19 | 20 | // AddToManager adds all Controllers to the Manager 21 | func AddToManager(mgr manager.Manager, opt *option.ControllersManagerOption, config *config.Config) error { 22 | if opt.EnableCluster { 23 | AddToManagerWithProviderFuncs = append(AddToManagerWithProviderFuncs, cluster.Add) 24 | } 25 | 26 | if opt.EnableMachine { 27 | AddToManagerWithProviderFuncs = append(AddToManagerWithProviderFuncs, machine.Add) 28 | } 29 | 30 | if opt.EnableAddons { 31 | AddToManagerWithProviderFuncs = append(AddToManagerWithProviderFuncs, addons.Add) 32 | } 33 | 34 | pMgr, err := provider.NewProvider(config) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | k8sMgr, _ := clustermanager.NewManager(clustermanager.ControlCluster{ 40 | Cluster: mgr, 41 | }) 42 | 43 | gMgr := &gmanager.GManager{ 44 | ProviderManager: pMgr, 45 | ClusterManager: k8sMgr, 46 | Config: config, 47 | } 48 | for _, f := range AddToManagerFuncs { 49 | if err := f(mgr); err != nil { 50 | return err 51 | } 52 | } 53 | 54 | for _, f := range AddToManagerWithProviderFuncs { 55 | if err := f(mgr, gMgr); err != nil { 56 | return err 57 | } 58 | } 59 | 60 | mgr.Add(gMgr.ClusterManager) 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /pkg/controllers/machine/machine_helper.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 8 | "github.com/wtxue/kok-operator/pkg/controllers/common" 9 | ) 10 | 11 | const ( 12 | machineClientRetryCount = 5 13 | machineClientRetryInterval = 5 * time.Second 14 | 15 | reasonFailedInit = "FailedInit" 16 | reasonFailedUpdate = "FailedUpdate" 17 | ) 18 | 19 | func (r *machineReconciler) onCreate(ctx *common.ClusterContext, machine *devopsv1.Machine) error { 20 | p, err := r.MpManager.GetProvider(ctx.Cluster.Spec.ClusterType) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | err = p.OnCreate(ctx, machine) 26 | if err != nil { 27 | machine.Status.Message = err.Error() 28 | machine.Status.Reason = reasonFailedInit 29 | r.Client.Status().Update(ctx.Ctx, machine) 30 | return err 31 | } 32 | 33 | condition := machine.Status.Conditions[len(machine.Status.Conditions)-1] 34 | if condition.Status == devopsv1.ConditionFalse { // means current condition run into error 35 | machine.Status.Message = condition.Message 36 | machine.Status.Reason = condition.Reason 37 | r.Client.Status().Update(ctx.Ctx, machine) 38 | return fmt.Errorf("Provider.OnCreate.%s [Failed] reason: %s message: %s", 39 | condition.Type, condition.Reason, condition.Message) 40 | } 41 | 42 | machine.Status.Message = "" 43 | machine.Status.Reason = "" 44 | err = r.Client.Status().Update(ctx.Ctx, machine) 45 | if err != nil { 46 | return err 47 | } 48 | return nil 49 | } 50 | 51 | func (r *machineReconciler) onUpdate(ctx *common.ClusterContext, machine *devopsv1.Machine) error { 52 | p, err := r.MpManager.GetProvider(ctx.Cluster.Spec.ClusterType) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | err = p.OnUpdate(ctx, machine) 58 | if err != nil { 59 | ctx.Cluster.Status.Message = err.Error() 60 | ctx.Cluster.Status.Reason = reasonFailedUpdate 61 | r.Client.Status().Update(ctx.Ctx, ctx.Cluster) 62 | return err 63 | } 64 | ctx.Cluster.Status.Message = "" 65 | ctx.Cluster.Status.Reason = "" 66 | r.Client.Status().Update(ctx.Ctx, ctx.Credential) 67 | r.Client.Status().Update(ctx.Ctx, ctx.Cluster) 68 | return nil 69 | } 70 | 71 | func (r *machineReconciler) reconcile(rc *manchineContext) error { 72 | ctx := &common.ClusterContext{ 73 | Ctx: rc.Ctx, 74 | Cluster: rc.Cluster, 75 | Credential: rc.ClusterCredential, 76 | Client: r.Client, 77 | ClusterManager: r.ClusterManager, 78 | Logger: rc.Logger, 79 | } 80 | 81 | var err error 82 | switch rc.Machine.Status.Phase { 83 | case devopsv1.MachineInitializing: 84 | rc.Logger.Info("onCreate") 85 | err = r.onCreate(ctx, rc.Machine) 86 | case devopsv1.MachineRunning: 87 | rc.Logger.Info("onUpdate") 88 | err = r.onUpdate(ctx, rc.Machine) 89 | default: 90 | err = fmt.Errorf("no handler for %q", rc.Cluster.Status.Phase) 91 | } 92 | 93 | return err 94 | } 95 | -------------------------------------------------------------------------------- /pkg/gmanager/gmanager.go: -------------------------------------------------------------------------------- 1 | package gmanager 2 | 3 | import ( 4 | "github.com/wtxue/kok-operator/pkg/clustermanager" 5 | "github.com/wtxue/kok-operator/pkg/provider" 6 | "github.com/wtxue/kok-operator/pkg/provider/config" 7 | ) 8 | 9 | type GManager struct { 10 | *provider.ProviderManager 11 | *clustermanager.ClusterManager 12 | *config.Config 13 | } 14 | -------------------------------------------------------------------------------- /pkg/k8sclient/client.go: -------------------------------------------------------------------------------- 1 | package k8sclient 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/user" 7 | "path/filepath" 8 | 9 | "github.com/pkg/errors" 10 | "k8s.io/client-go/kubernetes" 11 | "k8s.io/client-go/rest" 12 | "k8s.io/client-go/tools/clientcmd" 13 | "k8s.io/client-go/tools/clientcmd/api" 14 | ) 15 | 16 | // GetConfigWithContext returns kubernetes config based on the current environment. 17 | // If fpath is provided, loads configuration from that file. Otherwise, 18 | // GetConfig uses default strategy to load configuration from $KUBECONFIG, 19 | // .kube/config, or just returns in-cluster config. 20 | func GetConfigWithContext(kubeconfigPath, kubeContext string) (*rest.Config, error) { 21 | rules := clientcmd.NewDefaultClientConfigLoadingRules() 22 | if kubeconfigPath != "" { 23 | rules.ExplicitPath = kubeconfigPath 24 | } 25 | overrides := &clientcmd.ConfigOverrides{CurrentContext: kubeContext} 26 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() 27 | } 28 | 29 | // GetRawConfig creates a raw clientcmd api config 30 | func GetRawConfig(kubeconfigPath, kubeContext string) (api.Config, error) { 31 | rules := clientcmd.NewDefaultClientConfigLoadingRules() 32 | if kubeconfigPath != "" { 33 | rules.ExplicitPath = kubeconfigPath 34 | } 35 | overrides := &clientcmd.ConfigOverrides{CurrentContext: kubeContext} 36 | clientConfig := clientcmd. 37 | NewNonInteractiveDeferredLoadingClientConfig(rules, overrides) 38 | return clientConfig.RawConfig() 39 | } 40 | 41 | func GetConfig(kubeconfigPath string) (*rest.Config, error) { 42 | // If a flag is specified with the config location, use that 43 | if len(kubeconfigPath) > 0 { 44 | return clientcmd.BuildConfigFromFlags("", kubeconfigPath) 45 | } 46 | // If an env variable is specified with the config locaiton, use that 47 | if len(os.Getenv("KUBECONFIG")) > 0 { 48 | return clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) 49 | } 50 | // If no explicit location, try the in-cluster config 51 | if c, err := rest.InClusterConfig(); err == nil { 52 | return c, nil 53 | } 54 | // If no in-cluster config, try the default location in the user's home directory 55 | if usr, err := user.Current(); err == nil { 56 | if c, err := clientcmd.BuildConfigFromFlags( 57 | "", filepath.Join(usr.HomeDir, ".kube", "config")); err == nil { 58 | return c, nil 59 | } 60 | } 61 | 62 | return nil, fmt.Errorf("could not locate a kubeconfig") 63 | } 64 | 65 | // NewClientConfig creates a Kubernetes client config from raw kube config. 66 | func NewClientConfig(kubeConfig []byte) (*rest.Config, error) { 67 | if kubeConfig == nil { 68 | return nil, errors.New("kube config is empty") 69 | } 70 | apiconfig, err := clientcmd.Load(kubeConfig) 71 | if err != nil { 72 | return nil, errors.Wrap(err, "failed to load kubernetes API config") 73 | } 74 | 75 | clientConfig := clientcmd.NewDefaultClientConfig(*apiconfig, &clientcmd.ConfigOverrides{}) 76 | cfg, err := clientConfig.ClientConfig() 77 | if err != nil { 78 | return nil, errors.Wrap(err, "failed to build client config from API config") 79 | } 80 | 81 | // Adjust our client's rate limits based on the number of controllers we are running. 82 | if cfg.QPS == 0.0 { 83 | cfg.QPS = 40.0 84 | cfg.Burst = 60 85 | } 86 | 87 | return cfg, nil 88 | } 89 | 90 | // NewClientFromConfig creates a new Kubernetes client from config. 91 | func NewClientFromConfig(config *rest.Config) (kubernetes.Interface, error) { 92 | client, err := kubernetes.NewForConfig(config) 93 | if err != nil { 94 | return nil, errors.Wrap(err, "failed to create client for config") 95 | } 96 | 97 | return client, nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/k8sclient/scheme.go: -------------------------------------------------------------------------------- 1 | package k8sclient 2 | 3 | import ( 4 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 5 | workloadv1 "github.com/wtxue/kok-operator/pkg/apis/workload/v1" 6 | apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 9 | apiregistrationscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 10 | // monitorv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" 11 | ) 12 | 13 | var ( 14 | scheme = runtime.NewScheme() 15 | ) 16 | 17 | func init() { 18 | _ = clientgoscheme.AddToScheme(scheme) 19 | _ = apiextensionsscheme.AddToScheme(scheme) 20 | _ = apiregistrationscheme.AddToScheme(scheme) 21 | _ = devopsv1.AddToScheme(scheme) 22 | _ = workloadv1.AddToScheme(scheme) 23 | // _ = monitorv1.AddToScheme(scheme) 24 | } 25 | 26 | // GetScheme gets an initialized runtime.Scheme with k8s core added by default 27 | func GetScheme() *runtime.Scheme { 28 | return scheme 29 | } 30 | -------------------------------------------------------------------------------- /pkg/k8sutil/crds.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "time" 7 | 8 | apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 9 | apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 10 | apierrors "k8s.io/apimachinery/pkg/api/errors" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/util/wait" 13 | "k8s.io/client-go/rest" 14 | logf "sigs.k8s.io/controller-runtime/pkg/log" 15 | ) 16 | 17 | var logger = logf.Log.WithName("crd") 18 | 19 | // nolint 20 | // ReconcileCRDs crds apply must before manager cache start, so crds apply need use raw kubernetes client 21 | func ReconcileCRDs(cfg *rest.Config, crds []*apiextensionsv1.CustomResourceDefinition) error { 22 | ctx := context.TODO() 23 | cli := apiextensionsclient.NewForConfigOrDie(cfg) 24 | 25 | for _, crd := range crds { 26 | existing, err := cli.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{}) 27 | if err != nil { 28 | if apierrors.IsNotFound(err) { 29 | _, err = cli.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{}) 30 | if err != nil { 31 | return err 32 | } 33 | logger.Info("create successfully", "name", crd.Name, "kind", crd.Spec.Names.Kind) 34 | } else { 35 | return err 36 | } 37 | } else { 38 | // skip Conversion 39 | existing.Spec.Conversion = nil 40 | if same := reflect.DeepEqual(crd.Spec, existing.Spec); !same { 41 | crd.SetResourceVersion(existing.GetResourceVersion()) 42 | _, err = cli.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{}) 43 | if err != nil { 44 | return err 45 | } 46 | logger.Info("update successfully", "name", crd.Name, "kind", crd.Spec.Names.Kind) 47 | } else { 48 | logger.Info("update unchanges, ignored", "name", crd.Name, "kind", crd.Spec.Names.Kind) 49 | } 50 | } 51 | } 52 | 53 | backoff := wait.Backoff{ 54 | Duration: 2 * time.Second, 55 | Factor: 2, 56 | Jitter: 0.1, 57 | Steps: 5, 58 | } 59 | 60 | for _, crd := range crds { 61 | err := wait.ExponentialBackoff(backoff, func() (bool, error) { 62 | existing, err := cli.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{}) 63 | if err != nil { 64 | return false, err 65 | } 66 | 67 | for i := range existing.Status.Conditions { 68 | cond := &existing.Status.Conditions[i] 69 | if cond.Type == apiextensionsv1.Established && 70 | cond.Status == apiextensionsv1.ConditionTrue { 71 | return true, nil 72 | } 73 | } 74 | 75 | logger.Info("wait established ... ", "name", crd.Name, "kind", crd.Spec.Names.Kind) 76 | return false, nil 77 | }) 78 | 79 | if err != nil { 80 | return err 81 | } 82 | } 83 | 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/k8sutil/util_test.go: -------------------------------------------------------------------------------- 1 | package k8sutil 2 | 3 | import "testing" 4 | 5 | func TestGetServiceCIDRAndNodeCIDRMaskSize(t *testing.T) { 6 | type args struct { 7 | clusterCIDR string 8 | maxClusterServiceNum int32 9 | maxNodePodNum int32 10 | } 11 | tests := []struct { 12 | name string 13 | args args 14 | want string 15 | want1 int32 16 | wantErr bool 17 | }{ 18 | { 19 | name: "maxClusterServiceNum == 0", 20 | args: args{ 21 | clusterCIDR: "192.168.0.0/24", 22 | maxClusterServiceNum: 0, 23 | }, 24 | wantErr: true, 25 | }, 26 | { 27 | name: "maxNodePodNum == 0", 28 | args: args{ 29 | clusterCIDR: "192.168.0.0/24", 30 | maxClusterServiceNum: 0, 31 | }, 32 | wantErr: true, 33 | }, 34 | { 35 | name: "maxClusterServiceNum maxNodePodNum < clusterCIDR size", 36 | args: args{ 37 | clusterCIDR: "192.168.0.0/24", 38 | maxClusterServiceNum: 32, 39 | maxNodePodNum: 64, 40 | }, 41 | want: "192.168.0.224/27", 42 | want1: 26, 43 | wantErr: false, 44 | }, 45 | { 46 | name: "maxClusterServiceNum == clusterCIDR size", 47 | args: args{ 48 | clusterCIDR: "192.168.0.0/24", 49 | maxClusterServiceNum: 256, 50 | maxNodePodNum: 64, 51 | }, 52 | want: "192.168.0.0/24", 53 | want1: 26, 54 | wantErr: false, 55 | }, 56 | { 57 | name: "maxClusterServiceNum > clusterCIDR size", 58 | args: args{ 59 | clusterCIDR: "192.168.0.0/24", 60 | maxClusterServiceNum: 257, 61 | maxNodePodNum: 64, 62 | }, 63 | wantErr: true, 64 | }, 65 | } 66 | for _, tt := range tests { 67 | t.Run(tt.name, func(t *testing.T) { 68 | got, got1, err := GetServiceCIDRAndNodeCIDRMaskSize(tt.args.clusterCIDR, tt.args.maxClusterServiceNum, tt.args.maxNodePodNum) 69 | if (err != nil) != tt.wantErr { 70 | t.Errorf("GetServiceCIDRAndNodeCIDRMaskSize() error = %v, wantErr %v", err, tt.wantErr) 71 | return 72 | } 73 | if got != tt.want { 74 | t.Errorf("GetServiceCIDRAndNodeCIDRMaskSize() got = %v, want %v", got, tt.want) 75 | } 76 | if got1 != tt.want1 { 77 | t.Errorf("GetServiceCIDRAndNodeCIDRMaskSize() got1 = %v, want %v", got1, tt.want1) 78 | } 79 | }) 80 | } 81 | } 82 | 83 | func TestGetNodeCIDRMaskSize(t *testing.T) { 84 | type args struct { 85 | clusterCIDR string 86 | maxNodePodNum int32 87 | } 88 | tests := []struct { 89 | name string 90 | args args 91 | want int32 92 | wantErr bool 93 | }{ 94 | { 95 | name: "maxNodePodNum == 0", 96 | args: args{ 97 | clusterCIDR: "192.168.0.0/24", 98 | }, 99 | wantErr: true, 100 | }, 101 | { 102 | name: "maxNodePodNum < clusterCIDR size", 103 | args: args{ 104 | clusterCIDR: "192.168.0.0/24", 105 | maxNodePodNum: 64, 106 | }, 107 | want: 26, 108 | wantErr: false, 109 | }, 110 | { 111 | name: "maxNodePodNum == clusterCIDR size", 112 | args: args{ 113 | clusterCIDR: "192.168.0.0/24", 114 | maxNodePodNum: 256, 115 | }, 116 | want: 24, 117 | wantErr: false, 118 | }, 119 | { 120 | name: "maxNodePodNum > clusterCIDR size", 121 | args: args{ 122 | clusterCIDR: "192.168.0.0/25", 123 | maxNodePodNum: 256, 124 | }, 125 | wantErr: true, 126 | }, 127 | } 128 | for _, tt := range tests { 129 | t.Run(tt.name, func(t *testing.T) { 130 | got, err := GetNodeCIDRMaskSize(tt.args.clusterCIDR, tt.args.maxNodePodNum) 131 | if (err != nil) != tt.wantErr { 132 | t.Errorf("GetNodeCIDRMaskSize() error = %v, wantErr %v", err, tt.wantErr) 133 | return 134 | } 135 | if got != tt.want { 136 | t.Errorf("GetNodeCIDRMaskSize() got = %v, want %v", got, tt.want) 137 | } 138 | }) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /pkg/option/apiserver.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/spf13/pflag" 5 | ) 6 | 7 | type ApiServerOption struct { 8 | IsLocalKube bool 9 | 10 | // local / external 11 | IsLocalEtcd bool 12 | 13 | BaseBinDir string 14 | 15 | RootDir string 16 | 17 | // Endpoints of etcd members. Required for ExternalEtcd. 18 | Endpoints []string `json:"endpoints" protobuf:"bytes,1,rep,name=endpoints"` 19 | 20 | // CAFile is an SSL Certificate Authority file used to secure etcd communication. 21 | // Required if using a TLS connection. 22 | CAFile string `json:"caFile" protobuf:"bytes,2,opt,name=caFile"` 23 | 24 | // CertFile is an SSL certification file used to secure etcd communication. 25 | // Required if using a TLS connection. 26 | CertFile string `json:"certFile" protobuf:"bytes,3,opt,name=certFile"` 27 | 28 | // KeyFile is an SSL key file used to secure etcd communication. 29 | // Required if using a TLS connection. 30 | KeyFile string `json:"keyFile" protobuf:"bytes,4,opt,name=keyFile"` 31 | } 32 | 33 | func DefaultApiServerOption() *ApiServerOption { 34 | return &ApiServerOption{ 35 | IsLocalKube: true, 36 | IsLocalEtcd: true, 37 | BaseBinDir: "", 38 | RootDir: "/k8s", 39 | } 40 | } 41 | 42 | func (o *ApiServerOption) AddFlags(fs *pflag.FlagSet) { 43 | fs.BoolVar(&o.IsLocalKube, "is-local-kube", o.IsLocalKube, "enable local mock kube api server") 44 | fs.BoolVar(&o.IsLocalEtcd, "is-local-etcd", o.IsLocalEtcd, "when enable local mock kube use loacl etcd cluster") 45 | fs.StringVar(&o.BaseBinDir, "baseBinDir", o.BaseBinDir, "the base bin dir") 46 | fs.StringVar(&o.RootDir, "rootDir", o.RootDir, "the root bin dir") 47 | } 48 | -------------------------------------------------------------------------------- /pkg/option/controller.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/spf13/pflag" 5 | ) 6 | 7 | type ControllersManagerOption struct { 8 | EnableManagerCrds bool 9 | EnableCluster bool 10 | EnableMachine bool 11 | EnableAddons bool 12 | } 13 | 14 | func DefaultControllersManagerOption() *ControllersManagerOption { 15 | return &ControllersManagerOption{ 16 | EnableCluster: true, 17 | EnableMachine: true, 18 | EnableAddons: true, 19 | EnableManagerCrds: true, 20 | } 21 | } 22 | 23 | func (o *ControllersManagerOption) AddFlags(fs *pflag.FlagSet) { 24 | fs.BoolVar(&o.EnableManagerCrds, "enable-manager-crds", o.EnableManagerCrds, "Enables to manager the associated crds") 25 | fs.BoolVar(&o.EnableCluster, "enable-cluster", o.EnableCluster, "Enables the Cluster controller manager") 26 | fs.BoolVar(&o.EnableMachine, "enable-machine", o.EnableMachine, "Enables the Machine controller manager") 27 | fs.BoolVar(&o.EnableAddons, "enable-addons", o.EnableAddons, "Enables the cluster addons controller manager") 28 | } 29 | -------------------------------------------------------------------------------- /pkg/provider/baremetal/baremetal.go: -------------------------------------------------------------------------------- 1 | package baremetal 2 | 3 | const ( 4 | ProviderName = "baremetal" 5 | ) 6 | -------------------------------------------------------------------------------- /pkg/provider/baremetal/cluster/handler.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | func (p *Provider) ping(resp http.ResponseWriter, req *http.Request) { 9 | fmt.Fprint(resp, "pong") 10 | } 11 | -------------------------------------------------------------------------------- /pkg/provider/baremetal/cluster/update.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/thoas/go-funk" 10 | "github.com/wtxue/kok-operator/pkg/constants" 11 | "github.com/wtxue/kok-operator/pkg/controllers/common" 12 | "github.com/wtxue/kok-operator/pkg/k8sutil" 13 | "github.com/wtxue/kok-operator/pkg/provider/phases/certs" 14 | "github.com/wtxue/kok-operator/pkg/provider/phases/kubeadm" 15 | "github.com/wtxue/kok-operator/pkg/provider/phases/kubemisc" 16 | certutil "k8s.io/client-go/util/cert" 17 | ) 18 | 19 | func (p *Provider) EnsureRenewCerts(ctx *common.ClusterContext) error { 20 | for _, machine := range ctx.Cluster.Spec.Machines { 21 | s, err := machine.SSH() 22 | if err != nil { 23 | return err 24 | } 25 | 26 | data, err := s.ReadFile(constants.APIServerCertName) 27 | if err != nil { 28 | return err 29 | } 30 | cts, err := certutil.ParseCertsPEM(data) 31 | if err != nil { 32 | return err 33 | } 34 | expirationDuration := time.Until(cts[0].NotAfter) 35 | if expirationDuration > constants.RenewCertsTimeThreshold { 36 | ctx.Info("skip EnsureRenewCerts because expiration duration > threshold", 37 | "duration", expirationDuration, "threshold", constants.RenewCertsTimeThreshold) 38 | return nil 39 | } 40 | 41 | ctx.Info("EnsureRenewCerts", "node", s.Host) 42 | err = kubeadm.RenewCerts(s) 43 | if err != nil { 44 | return errors.Wrapf(err, "renew certs node: %s", machine.IP) 45 | } 46 | } 47 | 48 | return nil 49 | } 50 | 51 | func (p *Provider) EnsureAPIServerCert(ctx *common.ClusterContext) error { 52 | apiserver := certs.BuildApiserverEndpoint(ctx.Cluster.Spec.PublicAlternativeNames[0], kubemisc.GetBindPort(ctx.Cluster)) 53 | 54 | kubeadmConfig := kubeadm.GetKubeadmConfig(ctx, p.Cfg, apiserver) 55 | exptectCertSANs := k8sutil.GetAPIServerCertSANs(ctx.Cluster) 56 | 57 | needUpload := false 58 | for _, machine := range ctx.Cluster.Spec.Machines { 59 | sh, err := machine.SSH() 60 | if err != nil { 61 | return err 62 | } 63 | 64 | data, err := sh.ReadFile(constants.APIServerCertName) 65 | if err != nil { 66 | return err 67 | } 68 | certList, err := certutil.ParseCertsPEM(data) 69 | if err != nil { 70 | return err 71 | } 72 | actualCertSANs := certList[0].DNSNames 73 | for _, ip := range certList[0].IPAddresses { 74 | actualCertSANs = append(actualCertSANs, ip.String()) 75 | } 76 | 77 | if reflect.DeepEqual(funk.IntersectString(actualCertSANs, exptectCertSANs), exptectCertSANs) { 78 | return nil 79 | } 80 | 81 | ctx.Info("EnsureAPIServerCert", "node", sh.Host) 82 | for _, file := range []string{constants.APIServerCertName, constants.APIServerKeyName} { 83 | sh.CombinedOutput(fmt.Sprintf("rm -f %s", file)) 84 | } 85 | 86 | err = kubeadm.Init(ctx, sh, kubeadmConfig, "certs apiserver") 87 | if err != nil { 88 | return errors.Wrap(err, machine.IP) 89 | } 90 | err = kubeadm.RestartContainerByFilter(sh, kubeadm.DockerFilterForControlPlane("kube-apiserver")) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | needUpload = true 96 | } 97 | 98 | if needUpload { 99 | err := p.EnsureKubeadmInitUploadConfigPhase(ctx) 100 | if err != nil { 101 | return err 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /pkg/provider/baremetal/machine/provider.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import ( 4 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 5 | "github.com/wtxue/kok-operator/pkg/provider/baremetal" 6 | "github.com/wtxue/kok-operator/pkg/provider/baremetal/validation" 7 | "github.com/wtxue/kok-operator/pkg/provider/config" 8 | machineprovider "github.com/wtxue/kok-operator/pkg/provider/machine" 9 | "k8s.io/apimachinery/pkg/util/validation/field" 10 | ) 11 | 12 | func Add(mgr *machineprovider.MpManager, cfg *config.Config) error { 13 | p, err := NewProvider(mgr, cfg) 14 | if err != nil { 15 | return err 16 | } 17 | mgr.Register(p.Name(), p) 18 | return nil 19 | } 20 | 21 | type Provider struct { 22 | *machineprovider.DelegateProvider 23 | Mgr *machineprovider.MpManager 24 | Cfg *config.Config 25 | } 26 | 27 | func NewProvider(mgr *machineprovider.MpManager, cfg *config.Config) (*Provider, error) { 28 | p := &Provider{ 29 | Mgr: mgr, 30 | Cfg: cfg, 31 | } 32 | 33 | p.DelegateProvider = &machineprovider.DelegateProvider{ 34 | ProviderName: baremetal.ProviderName, 35 | CreateHandlers: []machineprovider.Handler{ 36 | p.EnsureCopyFiles, 37 | p.EnsurePreInstallHook, 38 | p.EnsureClean, 39 | p.EnsureRegistryHosts, 40 | 41 | p.EnsureEth, 42 | p.EnsureSystem, 43 | p.EnsureK8sComponent, 44 | p.EnsurePreflight, // wait basic setting done 45 | 46 | p.EnsureJoinNode, 47 | p.EnsureKubeconfig, 48 | p.EnsureMarkNode, 49 | p.EnsureCni, 50 | p.EnsureNodeReady, 51 | 52 | p.EnsurePostInstallHook, 53 | }, 54 | UpdateHandlers: []machineprovider.Handler{ 55 | p.EnsureCni, 56 | p.EnsurePostInstallHook, 57 | p.EnsureRegistryHosts, 58 | }, 59 | } 60 | 61 | return p, nil 62 | } 63 | 64 | var _ machineprovider.Provider = &Provider{} 65 | 66 | func (p *Provider) Validate(machine *devopsv1.Machine) field.ErrorList { 67 | return validation.ValidateMachine(machine) 68 | } 69 | -------------------------------------------------------------------------------- /pkg/provider/baremetal/validation/machine.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 5 | "k8s.io/apimachinery/pkg/util/validation/field" 6 | ) 7 | 8 | // ValidateMachine validates a given machine. 9 | func ValidateMachine(machine *devopsv1.Machine) field.ErrorList { 10 | allErrs := field.ErrorList{} 11 | 12 | allErrs = append(allErrs, ValidateMachineSpec(&machine.Spec, field.NewPath("spec"))...) 13 | 14 | return allErrs 15 | } 16 | 17 | // ValidateMachineSpec validates a given machine spec. 18 | func ValidateMachineSpec(spec *devopsv1.MachineSpec, fldPath *field.Path) field.ErrorList { 19 | allErrs := field.ErrorList{} 20 | 21 | return allErrs 22 | } 23 | -------------------------------------------------------------------------------- /pkg/provider/cluster/cluster.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "sync" 7 | 8 | "k8s.io/apiserver/pkg/server/mux" 9 | ) 10 | 11 | type CpManager struct { 12 | sync.RWMutex 13 | Cp map[string]Provider 14 | } 15 | 16 | func New() *CpManager { 17 | return &CpManager{ 18 | Cp: make(map[string]Provider), 19 | } 20 | } 21 | 22 | // Register makes a provider available by the provided name. 23 | // If Register is called twice with the same name or if provider is nil, 24 | // it panics. 25 | func (p *CpManager) Register(name string, provider Provider) { 26 | p.Lock() 27 | defer p.Unlock() 28 | if provider == nil { 29 | panic("cluster: Register provider is nil") 30 | } 31 | if _, dup := p.Cp[name]; dup { 32 | panic("cluster: Register called twice for provider " + name) 33 | } 34 | p.Cp[name] = provider 35 | } 36 | 37 | // RegisterHandler register all provider's hanlder. 38 | func (p *CpManager) RegisterHandler(mux *mux.PathRecorderMux) { 39 | for _, p := range p.Cp { 40 | p.RegisterHandler(mux) 41 | } 42 | } 43 | 44 | // Providers returns a sorted list of the names of the registered providers. 45 | func (p *CpManager) Providers() []string { 46 | p.RLock() 47 | defer p.RUnlock() 48 | var list []string 49 | for name := range p.Cp { 50 | list = append(list, name) 51 | } 52 | sort.Strings(list) 53 | return list 54 | } 55 | 56 | // GetProvider returns provider by name 57 | func (p *CpManager) GetProvider(name string) (Provider, error) { 58 | p.RLock() 59 | defer p.RUnlock() 60 | provider, ok := p.Cp[name] 61 | if !ok { 62 | return nil, fmt.Errorf("cluster: unknown provider %q (forgotten import?)", name) 63 | 64 | } 65 | 66 | return provider, nil 67 | } 68 | -------------------------------------------------------------------------------- /pkg/provider/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "path" 7 | "strings" 8 | 9 | "github.com/spf13/pflag" 10 | "github.com/wtxue/kok-operator/pkg/constants" 11 | ) 12 | 13 | type Config struct { 14 | Registry Registry 15 | Audit Audit 16 | Feature Feature 17 | Kubelet Kubelet `yaml:"kubelet"` 18 | APIServer APIServer `yaml:"apiServer"` 19 | ControllerManager ControllerManager `yaml:"controllerManager"` 20 | Scheduler Scheduler `yaml:"scheduler"` 21 | SupportK8sVersion []string 22 | CustomRegistry string 23 | EnableCustomCert bool 24 | EnableCustomImages bool 25 | EnableOnKube bool 26 | EnableHostNetwork bool 27 | } 28 | 29 | type Registry struct { 30 | Prefix string 31 | IP string 32 | Domain string 33 | Namespace string 34 | } 35 | 36 | type Audit struct { 37 | Address string 38 | } 39 | 40 | type Feature struct { 41 | SkipConditions []string 42 | } 43 | 44 | type Kubelet struct { 45 | ExtraArgs map[string]string `yaml:"extraArgs"` 46 | } 47 | 48 | type APIServer struct { 49 | ExtraArgs map[string]string `yaml:"extraArgs"` 50 | } 51 | 52 | type ControllerManager struct { 53 | ExtraArgs map[string]string `yaml:"extraArgs"` 54 | } 55 | 56 | type Scheduler struct { 57 | ExtraArgs map[string]string `yaml:"extraArgs"` 58 | } 59 | 60 | func NewDefaultConfig() *Config { 61 | return &Config{ 62 | CustomRegistry: "registry.aliyuncs.com/google_containers", // "docker.io/wtxue" 63 | SupportK8sVersion: constants.K8sVersions, 64 | EnableOnKube: true, 65 | EnableCustomImages: false, 66 | EnableHostNetwork: false, 67 | } 68 | } 69 | 70 | func (r *Config) NeedSetHosts() bool { 71 | return r.Registry.IP != "" 72 | } 73 | 74 | func (r *Config) ImageFullName(name, tag string) string { 75 | b := new(bytes.Buffer) 76 | b.WriteString(name) 77 | if tag != "" { 78 | if !strings.Contains(tag, "v") { 79 | b.WriteString(":" + "v" + tag) 80 | } else { 81 | b.WriteString(":" + tag) 82 | } 83 | } 84 | 85 | s := strings.Split(r.CustomRegistry, "/") 86 | r.Registry.Domain = s[0] 87 | r.Registry.Namespace = s[1] 88 | return path.Join(r.Registry.Domain, r.Registry.Namespace, b.String()) 89 | } 90 | 91 | func (r *Config) KubeAllImageFullName(name, tag string) string { 92 | if !strings.Contains(tag, "v") { 93 | tag = "v" + tag 94 | } 95 | 96 | return fmt.Sprintf("%s/%s:%s", r.CustomRegistry, name, tag) 97 | } 98 | 99 | func (r *Config) KubeProxyImagesName(tag string) string { 100 | if !strings.Contains(tag, "v") { 101 | tag = "v" + tag 102 | } 103 | 104 | return fmt.Sprintf("%s/%s:%s", r.CustomRegistry, "kube-proxy", tag) 105 | } 106 | 107 | func (r *Config) IsK8sSupport(version string) bool { 108 | for _, v := range r.SupportK8sVersion { 109 | if v == version { 110 | return true 111 | } 112 | } 113 | 114 | return false 115 | } 116 | 117 | func (r *Config) AddFlags(fs *pflag.FlagSet) { 118 | fs.StringVar(&r.CustomRegistry, "images-prefix", r.CustomRegistry, "the images prefix") 119 | fs.BoolVar(&r.EnableOnKube, "enable-onkube", r.EnableOnKube, "if true, the cluster manager will use on kube apiserver") 120 | fs.BoolVar(&r.EnableHostNetwork, "enable-host-network", r.EnableHostNetwork, "if true, the kube-apiserver pod use hostNetwork") 121 | fs.BoolVar(&r.EnableCustomImages, "enable-custom-images", r.EnableCustomImages, "enable custom images") 122 | fs.StringArrayVar(&r.SupportK8sVersion, "support-k8s-version", r.SupportK8sVersion, "the support k8s version") 123 | } 124 | -------------------------------------------------------------------------------- /pkg/provider/machine/machine.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "sync" 7 | ) 8 | 9 | type MpManager struct { 10 | sync.RWMutex 11 | Mp map[string]Provider 12 | } 13 | 14 | func New() *MpManager { 15 | return &MpManager{ 16 | Mp: make(map[string]Provider), 17 | } 18 | } 19 | 20 | // Register makes a provider available by the provided name. 21 | // If Register is called twice with the same name or if provider is nil, it panics. 22 | func (p *MpManager) Register(name string, provider Provider) { 23 | p.Lock() 24 | defer p.Unlock() 25 | if provider == nil { 26 | panic("machine: Register provider is nil") 27 | } 28 | if _, dup := p.Mp[name]; dup { 29 | panic("machine: Register called twice for provider " + name) 30 | } 31 | p.Mp[name] = provider 32 | } 33 | 34 | // Providers returns a sorted list of the names of the registered providers. 35 | func (p *MpManager) Providers() []string { 36 | p.RLock() 37 | defer p.RUnlock() 38 | var list []string 39 | for name := range p.Mp { 40 | list = append(list, name) 41 | } 42 | sort.Strings(list) 43 | return list 44 | } 45 | 46 | // GetProvider returns provider by name 47 | func (p *MpManager) GetProvider(name string) (Provider, error) { 48 | p.RLock() 49 | defer p.RUnlock() 50 | provider, ok := p.Mp[name] 51 | if !ok { 52 | return nil, fmt.Errorf("machine: unknown provider %q (forgotten import?)", name) 53 | 54 | } 55 | 56 | return provider, nil 57 | } 58 | -------------------------------------------------------------------------------- /pkg/provider/managed/cluster/provider.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "path" 5 | "strings" 6 | 7 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 8 | "github.com/wtxue/kok-operator/pkg/constants" 9 | "github.com/wtxue/kok-operator/pkg/controllers/common" 10 | "github.com/wtxue/kok-operator/pkg/provider/baremetal/validation" 11 | clusterprovider "github.com/wtxue/kok-operator/pkg/provider/cluster" 12 | "github.com/wtxue/kok-operator/pkg/provider/config" 13 | "github.com/wtxue/kok-operator/pkg/provider/managed" 14 | "github.com/wtxue/kok-operator/pkg/util/pointer" 15 | "k8s.io/apimachinery/pkg/util/validation/field" 16 | "k8s.io/apiserver/pkg/server/mux" 17 | ) 18 | 19 | func Add(mgr *clusterprovider.CpManager, cfg *config.Config) error { 20 | p, err := NewProvider(mgr, cfg) 21 | if err != nil { 22 | return err 23 | } 24 | mgr.Register(p.Name(), p) 25 | return nil 26 | } 27 | 28 | type Provider struct { 29 | *clusterprovider.DelegateProvider 30 | Mgr *clusterprovider.CpManager 31 | Cfg *config.Config 32 | } 33 | 34 | var _ clusterprovider.Provider = &Provider{} 35 | 36 | func NewProvider(mgr *clusterprovider.CpManager, cfg *config.Config) (*Provider, error) { 37 | p := &Provider{ 38 | Mgr: mgr, 39 | Cfg: cfg, 40 | } 41 | 42 | p.DelegateProvider = &clusterprovider.DelegateProvider{ 43 | ProviderName: managed.ProviderName, 44 | CreateHandlers: []clusterprovider.Handler{ 45 | p.EnsurePreInstallHook, 46 | p.EnsureClusterComplete, 47 | p.EnsureEtcd, 48 | p.EnsureCerts, 49 | p.EnsureKubeMisc, 50 | p.EnsureKubeMaster, 51 | 52 | p.EnsureKubeconfig, 53 | p.EnsurePostInstallHook, 54 | }, 55 | UpdateHandlers: []clusterprovider.Handler{ 56 | p.EnsureKubeconfig, 57 | p.EnsureKubeMaster, 58 | p.EnsureAddons, 59 | p.EnsureCni, 60 | p.EnsureMetricsServer, 61 | }, 62 | } 63 | 64 | return p, nil 65 | } 66 | 67 | func (p *Provider) RegisterHandler(mux *mux.PathRecorderMux) { 68 | prefix := "/provider/" + strings.ToLower(p.Name()) 69 | 70 | mux.HandleFunc(path.Join(prefix, "ping"), p.ping) 71 | } 72 | 73 | func (p *Provider) Validate(ctx *common.ClusterContext) field.ErrorList { 74 | return validation.ValidateCluster(ctx) 75 | } 76 | 77 | func (p *Provider) PreCreate(ctx *common.ClusterContext) error { 78 | if ctx.Cluster.Spec.Version == "" { 79 | ctx.Cluster.Spec.Version = constants.K8sVersions[0] 80 | } 81 | if ctx.Cluster.Spec.ClusterCIDR == "" { 82 | ctx.Cluster.Spec.ClusterCIDR = "10.244.0.0/16" 83 | } 84 | if ctx.Cluster.Spec.NetworkDevice == "" { 85 | ctx.Cluster.Spec.NetworkDevice = "eth0" 86 | } 87 | 88 | if ctx.Cluster.Spec.Features.IPVS == nil { 89 | ctx.Cluster.Spec.Features.IPVS = pointer.ToBool(true) 90 | } 91 | 92 | if ctx.Cluster.Spec.Properties.MaxClusterServiceNum == nil && ctx.Cluster.Spec.ServiceCIDR == nil { 93 | ctx.Cluster.Spec.Properties.MaxClusterServiceNum = pointer.ToInt32(256) 94 | } 95 | if ctx.Cluster.Spec.Properties.MaxNodePodNum == nil { 96 | ctx.Cluster.Spec.Properties.MaxNodePodNum = pointer.ToInt32(256) 97 | } 98 | if ctx.Cluster.Spec.Features.SkipConditions == nil { 99 | ctx.Cluster.Spec.Features.SkipConditions = p.Cfg.Feature.SkipConditions 100 | } 101 | 102 | if ctx.Cluster.Spec.Etcd == nil { 103 | ctx.Cluster.Spec.Etcd = &devopsv1.Etcd{Local: &devopsv1.LocalEtcd{}} 104 | } 105 | 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /pkg/provider/managed/machine/provider.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import ( 4 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 5 | "github.com/wtxue/kok-operator/pkg/provider/baremetal/validation" 6 | "github.com/wtxue/kok-operator/pkg/provider/config" 7 | machineprovider "github.com/wtxue/kok-operator/pkg/provider/machine" 8 | "github.com/wtxue/kok-operator/pkg/provider/managed" 9 | "k8s.io/apimachinery/pkg/util/validation/field" 10 | ) 11 | 12 | func Add(mgr *machineprovider.MpManager, cfg *config.Config) error { 13 | p, err := NewProvider(mgr, cfg) 14 | if err != nil { 15 | return err 16 | } 17 | mgr.Register(p.Name(), p) 18 | return nil 19 | } 20 | 21 | type Provider struct { 22 | *machineprovider.DelegateProvider 23 | Mgr *machineprovider.MpManager 24 | Cfg *config.Config 25 | } 26 | 27 | func NewProvider(mgr *machineprovider.MpManager, cfg *config.Config) (*Provider, error) { 28 | p := &Provider{ 29 | Mgr: mgr, 30 | Cfg: cfg, 31 | } 32 | 33 | p.DelegateProvider = &machineprovider.DelegateProvider{ 34 | ProviderName: managed.ProviderName, 35 | CreateHandlers: []machineprovider.Handler{ 36 | p.EnsureCopyFiles, 37 | p.EnsurePreInstallHook, 38 | p.EnsureClean, 39 | p.EnsureRegistryHosts, 40 | 41 | p.EnsureEth, 42 | p.EnsureCRI, 43 | p.EnsureSystem, 44 | p.EnsureK8sComponent, 45 | p.EnsurePreflight, // wait basic setting done 46 | 47 | p.EnsureJoinNode, 48 | p.EnsureKubeconfig, 49 | p.EnsureMarkNode, 50 | p.EnsureCni, 51 | p.EnsureNodeReady, 52 | 53 | p.EnsurePostInstallHook, 54 | }, 55 | UpdateHandlers: []machineprovider.Handler{ 56 | p.EnsureCni, 57 | p.EnsurePostInstallHook, 58 | p.EnsureRegistryHosts, 59 | }, 60 | } 61 | 62 | return p, nil 63 | } 64 | 65 | var _ machineprovider.Provider = &Provider{} 66 | 67 | func (p *Provider) Validate(machine *devopsv1.Machine) field.ErrorList { 68 | return validation.ValidateMachine(machine) 69 | } 70 | -------------------------------------------------------------------------------- /pkg/provider/managed/managed.go: -------------------------------------------------------------------------------- 1 | package managed 2 | 3 | const ( 4 | ProviderName = "managed" 5 | ) 6 | -------------------------------------------------------------------------------- /pkg/provider/phases/certs/certs.go: -------------------------------------------------------------------------------- 1 | package certs 2 | 3 | import ( 4 | "crypto" 5 | "crypto/x509" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/wtxue/kok-operator/pkg/apis" 9 | "github.com/wtxue/kok-operator/pkg/util/pkiutil" 10 | logf "sigs.k8s.io/controller-runtime/pkg/log" 11 | ) 12 | 13 | type CaAll struct { 14 | CaCert *x509.Certificate 15 | CaKey crypto.Signer 16 | Cfg *KubeadmCert 17 | } 18 | 19 | // CreateCACertAndKeyFiles generates and writes out a given certificate authority. 20 | // The certSpec should be one of the variables from this package. 21 | func CreateCACertAndKeyFiles(certSpec *KubeadmCert, cfg *apis.WarpperConfiguration, cfgMaps map[string][]byte) (*CaAll, error) { 22 | if certSpec.CAName != "" { 23 | return nil, errors.Errorf("this function should only be used for CAs, but cert %s has CA %s", certSpec.Name, certSpec.CAName) 24 | } 25 | 26 | logf.Log.V(2).Info("creating a new certificate authority", "name", certSpec.Name) 27 | certConfig, err := certSpec.GetConfig(cfg) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | caCert, caKey, err := pkiutil.NewCertificateAuthority(certConfig) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | keyPath, keyByte, err := pkiutil.BuildKeyByte(cfg.CertificatesDir, certSpec.BaseName, caKey) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | cfgMaps[keyPath] = keyByte 43 | certPath, certByte, err := pkiutil.BuildCertByte(cfg.CertificatesDir, certSpec.BaseName, caCert) 44 | if err != nil { 45 | return nil, err 46 | } 47 | cfgMaps[certPath] = certByte 48 | 49 | return &CaAll{ 50 | CaCert: caCert, 51 | CaKey: caKey, 52 | Cfg: certSpec}, nil 53 | } 54 | 55 | func CreateCertAndKeyFilesWithCA(certSpec *KubeadmCert, ca *CaAll, cfg *apis.WarpperConfiguration, certsMaps map[string][]byte) error { 56 | if certSpec.CAName != ca.Cfg.Name { 57 | return errors.Errorf("expected CAname for %s to be %q, but was %s", certSpec.Name, certSpec.CAName, ca.Cfg.Name) 58 | } 59 | 60 | certConfig, err := certSpec.GetConfig(cfg) 61 | if err != nil { 62 | return errors.Wrapf(err, "couldn't create %q certificate", certSpec.Name) 63 | } 64 | 65 | cert, key, err := pkiutil.NewCertAndKey(ca.CaCert, ca.CaKey, certConfig) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | keyPath, keyByte, err := pkiutil.BuildKeyByte(cfg.CertificatesDir, certSpec.BaseName, key) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | certsMaps[keyPath] = keyByte 76 | certPath, certByte, err := pkiutil.BuildCertByte(cfg.CertificatesDir, certSpec.BaseName, cert) 77 | if err != nil { 78 | return err 79 | } 80 | certsMaps[certPath] = certByte 81 | return nil 82 | } 83 | 84 | // CreateServiceAccountKeyAndPublicKeyFiles creates new public/private key files for signing service account users. 85 | // If the sa public/private key files already exist in the target folder, they are used only if evaluated equals; otherwise an error is returned. 86 | func CreateServiceAccountKeyAndPublicKeyFiles(certsDir string, keyType x509.PublicKeyAlgorithm, certsMaps map[string][]byte) error { 87 | logf.Log.V(2).Info("creating new public/private key files for signing service account users") 88 | // The key does NOT exist, let's generate it now 89 | key, err := pkiutil.NewPrivateKey(keyType) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | // Write .key and .pub files to remote 95 | logf.Log.V(2).Info("[certs] Generating key and public key", "name", pkiutil.ServiceAccountKeyBaseName) 96 | keyPath, keyByte, err := pkiutil.BuildKeyByte(certsDir, pkiutil.ServiceAccountKeyBaseName, key) 97 | if err != nil { 98 | return err 99 | } 100 | certsMaps[keyPath] = keyByte 101 | 102 | publicPath, publicByte, err := pkiutil.BuildPublicKeyByte(certsDir, pkiutil.ServiceAccountKeyBaseName, key.Public()) 103 | if err != nil { 104 | return err 105 | } 106 | certsMaps[publicPath] = publicByte 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /pkg/provider/phases/clean/clean.go: -------------------------------------------------------------------------------- 1 | package clean 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/wtxue/kok-operator/pkg/util/ssh" 7 | 8 | "github.com/pkg/errors" 9 | logf "sigs.k8s.io/controller-runtime/pkg/log" 10 | ) 11 | 12 | func CleanNode(s ssh.Interface) error { 13 | cmd := "kubeadm reset -f && rm -rf /var/lib/etcd /var/lib/kubelet /var/lib/dockershim /var/run/kubernetes /var/lib/cni /etc/kubernetes /etc/cni /root/.kube /opt/k8s && ipvsadm --clear" 14 | logf.Log.V(4).Info("start exec", "node", s.HostIP(), "cmd", cmd) 15 | exit, err := s.ExecStream(cmd, os.Stdout, os.Stderr) 16 | if err != nil { 17 | logf.Log.Error(err, "exec err", "node", s.HostIP(), "cmd", cmd, "exit", exit) 18 | return errors.Wrapf(err, "node: %s exec: \n%s", s.HostIP(), cmd) 19 | } 20 | 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/provider/phases/gpu/gpu.go: -------------------------------------------------------------------------------- 1 | package gpu 2 | 3 | import ( 4 | "context" 5 | "github.com/wtxue/kok-operator/pkg/util/ssh" 6 | clientset "k8s.io/client-go/kubernetes" 7 | ) 8 | 9 | type NvidiaDriverOption struct { 10 | } 11 | 12 | func InstallNvidiaDriver(s ssh.Interface, option *NvidiaDriverOption) error { 13 | 14 | return nil 15 | } 16 | 17 | type NvidiaContainerRuntimeOption struct { 18 | } 19 | 20 | func InstallNvidiaContainerRuntime(s ssh.Interface, option *NvidiaContainerRuntimeOption) error { 21 | 22 | return nil 23 | } 24 | 25 | type NvidiaDevicePluginOption struct { 26 | Image string 27 | } 28 | 29 | func InstallNvidiaDevicePlugin(ctx context.Context, clientset clientset.Interface, option *NvidiaDevicePluginOption) error { 30 | 31 | return nil 32 | } 33 | 34 | func IsEnable(labels map[string]string) bool { 35 | return labels["nvidia-device-enable"] == "enable" 36 | } 37 | 38 | func MachineIsSupport(s ssh.Interface) bool { 39 | return true 40 | } 41 | -------------------------------------------------------------------------------- /pkg/provider/phases/kubebin/kubebin.go: -------------------------------------------------------------------------------- 1 | package kubebin 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 8 | "github.com/wtxue/kok-operator/pkg/constants" 9 | "github.com/wtxue/kok-operator/pkg/controllers/common" 10 | "github.com/wtxue/kok-operator/pkg/util/ssh" 11 | ) 12 | 13 | func Install(ctx *common.ClusterContext, s ssh.Interface) error { 14 | // dir := "bin/linux/" // local debug config dir 15 | k8sDir := fmt.Sprintf("/k8s-%s/bin/", ctx.Cluster.Spec.Version) 16 | otherDir := "/k8s/bin/" 17 | if dir := constants.GetMapKey(ctx.Cluster.Annotations, constants.ClusterDebugLocalDir); len(dir) > 0 { 18 | k8sDir = dir + k8sDir 19 | otherDir = dir + otherDir 20 | } 21 | 22 | var CopyList = []devopsv1.File{ 23 | { 24 | Src: k8sDir + "kubectl", 25 | Dst: "/usr/local/bin/kubectl", 26 | }, 27 | { 28 | Src: k8sDir + "kubeadm", 29 | Dst: "/usr/local/bin/kubeadm", 30 | }, 31 | { 32 | Src: otherDir + "k9s", 33 | Dst: "/usr/local/bin/k9s", 34 | }, 35 | { 36 | Src: k8sDir + "kubelet", 37 | Dst: "/usr/bin/kubelet", 38 | }, 39 | { 40 | Src: otherDir + "cni.tgz", 41 | Dst: "/opt/k8s/cni.tgz", 42 | }, 43 | } 44 | 45 | for _, ls := range CopyList { 46 | if ok, err := s.Exist(ls.Dst); err == nil && ok { 47 | ctx.Info("file exist ignoring", "node", s.HostIP(), "dst", ls.Dst) 48 | continue 49 | } 50 | 51 | err := s.CopyFile(ls.Src, ls.Dst) 52 | if err != nil { 53 | ctx.Error(err, "CopyFile", "node", s.HostIP(), "src", ls.Src) 54 | return err 55 | } 56 | 57 | if strings.Contains(ls.Dst, "bin") { 58 | _, _, _, err = s.Execf("chmod a+x %s", ls.Dst) 59 | if err != nil { 60 | return err 61 | } 62 | } 63 | 64 | if strings.Contains(ls.Dst, "cni") { 65 | cmd := fmt.Sprintf("mkdir -p %s && tar -C %s -xzf /opt/k8s/cni.tgz", constants.CNIBinDir, constants.CNIBinDir) 66 | _, err := s.CombinedOutput(cmd) 67 | if err != nil { 68 | return err 69 | } 70 | } 71 | 72 | ctx.Info("copy success", "node", s.HostIP(), "dst", ls.Dst) 73 | } 74 | 75 | ctx.Info("write kubelet systemd unit file", "node", s.HostIP(), "dst", constants.KubeletSystemdUnitFilePath) 76 | err := s.WriteFile(strings.NewReader(constants.KubeletService), constants.KubeletSystemdUnitFilePath) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | ctx.Info("write kubelet systemd service run config", "node", s.HostIP(), "path", constants.KubeletServiceRunConfig) 82 | err = s.WriteFile(strings.NewReader(constants.KubeletServiceRunConfig), constants.KubeletServiceRunConfigPath) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | cmd := "mkdir -p /etc/kubernetes/manifests/ && systemctl enable kubelet && systemctl daemon-reload && systemctl restart kubelet" 88 | if _, stderr, exit, err := s.Execf(cmd); err != nil || exit != 0 { 89 | cmd = "journalctl --unit kubelet -n10 --no-pager" 90 | jStdout, _, jExit, jErr := s.Execf(cmd) 91 | if jErr != nil || jExit != 0 { 92 | return fmt.Errorf("exec %q:error %s", cmd, err) 93 | } 94 | 95 | ctx.Info("log", "cmd", cmd, "stdout", jStdout) 96 | return fmt.Errorf("Exec %s failed:exit %d:stderr %s:error %s:log:\n%s", cmd, exit, stderr, err, jStdout) 97 | } 98 | ctx.Info("exec successfully", "node", s.HostIP(), "cmd", cmd) 99 | 100 | cmd = fmt.Sprintf("echo 'source <(kubectl completion bash)' >>~/.bashrc && kubectl completion bash > /etc/bash_completion.d/kubectl") 101 | _, err = s.CombinedOutput(cmd) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | ctx.Info("exec successfully", "node", s.HostIP(), "cmd", cmd) 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /pkg/provider/phases/kubemisc/helpers.go: -------------------------------------------------------------------------------- 1 | package kubemisc 2 | 3 | import ( 4 | "fmt" 5 | 6 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 7 | ) 8 | 9 | // CreateBasic creates a basic, general KubeConfig object that then can be extended 10 | func CreateBasic(serverURL, clusterName, userName string, caCert []byte) *clientcmdapi.Config { 11 | // Use the cluster and the username as the context name 12 | contextName := fmt.Sprintf("%s@%s", userName, clusterName) 13 | 14 | return &clientcmdapi.Config{ 15 | Clusters: map[string]*clientcmdapi.Cluster{ 16 | clusterName: { 17 | Server: serverURL, 18 | CertificateAuthorityData: caCert, 19 | }, 20 | }, 21 | Contexts: map[string]*clientcmdapi.Context{ 22 | contextName: { 23 | Cluster: clusterName, 24 | AuthInfo: userName, 25 | }, 26 | }, 27 | AuthInfos: map[string]*clientcmdapi.AuthInfo{}, 28 | CurrentContext: contextName, 29 | } 30 | } 31 | 32 | // CreateWithCerts creates a KubeConfig object with access to the API server with client certificates 33 | func CreateWithCerts(serverURL, clusterName, userName string, caCert []byte, clientKey []byte, clientCert []byte) *clientcmdapi.Config { 34 | config := CreateBasic(serverURL, clusterName, userName, caCert) 35 | config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ 36 | ClientKeyData: clientKey, 37 | ClientCertificateData: clientCert, 38 | } 39 | return config 40 | } 41 | 42 | // CreateWithToken creates a KubeConfig object with access to the API server with a token 43 | func CreateWithToken(serverURL, clusterName, userName string, caCert []byte, token string) *clientcmdapi.Config { 44 | config := CreateBasic(serverURL, clusterName, userName, caCert) 45 | config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ 46 | Token: token, 47 | } 48 | return config 49 | } 50 | -------------------------------------------------------------------------------- /pkg/provider/phases/system/system.go: -------------------------------------------------------------------------------- 1 | package system 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/pkg/errors" 10 | devopsv1 "github.com/wtxue/kok-operator/pkg/apis/devops/v1" 11 | "github.com/wtxue/kok-operator/pkg/constants" 12 | "github.com/wtxue/kok-operator/pkg/controllers/common" 13 | "github.com/wtxue/kok-operator/pkg/util/ssh" 14 | "github.com/wtxue/kok-operator/pkg/util/template" 15 | ) 16 | 17 | type Option struct { 18 | // InsecureRegistries string 19 | // RegistryMirrors string // "https://yqdzw3p0.mirror.aliyuncs.com" 20 | // RegistryDomain string // "download.docker.com" or "mirrors.aliyun.com" 21 | Options string 22 | K8sVersion string 23 | // ContainerdVersion string 24 | // Cgroupdriver string // cgroupfs or systemd 25 | HostIP string 26 | KernelRepo string 27 | ResolvConf string 28 | CentosVersion string 29 | ExtraArgs map[string]string 30 | } 31 | 32 | func shellTemplate(ctx *common.ClusterContext) string { 33 | switch ctx.Cluster.Spec.OSType { 34 | case devopsv1.DebianType: 35 | return debianShellTemplate 36 | case devopsv1.UbuntuType: 37 | return ubuntuShellTemplate 38 | default: 39 | return centosShellTemplate 40 | } 41 | } 42 | 43 | func Install(ctx *common.ClusterContext, s ssh.Interface) error { 44 | option := &Option{ 45 | K8sVersion: ctx.Cluster.Spec.Version, 46 | HostIP: s.HostIP(), 47 | } 48 | 49 | // _, _, _, err := s.Execf("hostnamectl set-hostname %s", s.HostIP()) 50 | // if err != nil { 51 | // return err 52 | // } 53 | 54 | initData, err := template.ParseString(shellTemplate(ctx), option) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | err = s.WriteFile(bytes.NewReader(initData), constants.SystemInitFile) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | ctx.Info("start exec init system ... ", "node", option.HostIP) 65 | cmd := fmt.Sprintf("chmod a+x %s && %s", constants.SystemInitFile, constants.SystemInitFile) 66 | exit, err := s.ExecStream(cmd, os.Stdout, os.Stderr) 67 | if err != nil { 68 | ctx.Error(err, "exit", exit, "node", option.HostIP) 69 | return errors.Wrapf(err, "node: %s exec init", option.HostIP) 70 | } 71 | 72 | ctx.Info("system init successfully", "node", option.HostIP) 73 | return nil 74 | } 75 | 76 | func CopyFile(s ssh.Interface, file *devopsv1.File) error { 77 | if ok, err := s.Exist(file.Dst); err == nil && ok { 78 | return nil 79 | } 80 | 81 | err := s.CopyFile(file.Src, file.Dst) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | if strings.Contains(file.Dst, "bin") { 87 | _, _, _, err = s.Execf("chmod a+x %s", file.Dst) 88 | if err != nil { 89 | return err 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /pkg/provider/provider.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | baremetalcluster "github.com/wtxue/kok-operator/pkg/provider/baremetal/cluster" 5 | baremetalmachine "github.com/wtxue/kok-operator/pkg/provider/baremetal/machine" 6 | "github.com/wtxue/kok-operator/pkg/provider/cluster" 7 | clusterprovider "github.com/wtxue/kok-operator/pkg/provider/cluster" 8 | "github.com/wtxue/kok-operator/pkg/provider/config" 9 | "github.com/wtxue/kok-operator/pkg/provider/machine" 10 | machineprovider "github.com/wtxue/kok-operator/pkg/provider/machine" 11 | managedcluster "github.com/wtxue/kok-operator/pkg/provider/managed/cluster" 12 | managedmachine "github.com/wtxue/kok-operator/pkg/provider/managed/machine" 13 | ) 14 | 15 | type ProviderManager struct { 16 | *cluster.CpManager 17 | *machine.MpManager 18 | Cfg *config.Config 19 | } 20 | 21 | var AddToCpManagerFuncs []func(*clusterprovider.CpManager, *config.Config) error 22 | var AddToMpManagerFuncs []func(*machineprovider.MpManager, *config.Config) error 23 | 24 | func NewProvider(config *config.Config) (*ProviderManager, error) { 25 | AddToCpManagerFuncs = append(AddToCpManagerFuncs, baremetalcluster.Add) 26 | AddToCpManagerFuncs = append(AddToCpManagerFuncs, managedcluster.Add) 27 | 28 | AddToMpManagerFuncs = append(AddToMpManagerFuncs, baremetalmachine.Add) 29 | AddToMpManagerFuncs = append(AddToMpManagerFuncs, managedmachine.Add) 30 | 31 | mgr := &ProviderManager{ 32 | CpManager: cluster.New(), 33 | MpManager: machine.New(), 34 | Cfg: config, 35 | } 36 | 37 | for _, f := range AddToCpManagerFuncs { 38 | if err := f(mgr.CpManager, mgr.Cfg); err != nil { 39 | return nil, err 40 | } 41 | } 42 | 43 | for _, f := range AddToMpManagerFuncs { 44 | if err := f(mgr.MpManager, mgr.Cfg); err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | return mgr, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/static/crds.go: -------------------------------------------------------------------------------- 1 | package static 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "embed" 7 | "fmt" 8 | "io" 9 | "strings" 10 | 11 | "github.com/wtxue/kok-operator/pkg/constants" 12 | "github.com/wtxue/kok-operator/pkg/k8sclient" 13 | 14 | apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 15 | "k8s.io/apimachinery/pkg/runtime/schema" 16 | "k8s.io/apimachinery/pkg/runtime/serializer/json" 17 | ) 18 | 19 | //go:embed crds/* 20 | var fcrds embed.FS 21 | 22 | func load(f io.Reader) ([]*apiextensionsv1.CustomResourceDefinition, error) { 23 | var b bytes.Buffer 24 | 25 | var yamls []string 26 | 27 | crds := make([]*apiextensionsv1.CustomResourceDefinition, 0) 28 | scanner := bufio.NewScanner(f) 29 | for scanner.Scan() { 30 | line := scanner.Text() 31 | if line == "---" { 32 | yamls = append(yamls, b.String()) 33 | b.Reset() 34 | } else { 35 | if _, err := b.WriteString(line); err != nil { 36 | return crds, err 37 | } 38 | if _, err := b.WriteString("\n"); err != nil { 39 | return crds, err 40 | } 41 | } 42 | } 43 | if s := strings.TrimSpace(b.String()); s != "" { 44 | yamls = append(yamls, s) 45 | } 46 | 47 | for _, yaml := range yamls { 48 | s := json.NewSerializerWithOptions(json.DefaultMetaFactory, 49 | k8sclient.GetScheme(), k8sclient.GetScheme(), json.SerializerOptions{Yaml: true}) 50 | 51 | obj, _, err := s.Decode([]byte(yaml), nil, nil) 52 | if err != nil { 53 | continue 54 | } 55 | 56 | var crd *apiextensionsv1.CustomResourceDefinition 57 | var ok bool 58 | if crd, ok = obj.(*apiextensionsv1.CustomResourceDefinition); !ok { 59 | continue 60 | } 61 | 62 | crd.Status = apiextensionsv1.CustomResourceDefinitionStatus{} 63 | crd.SetGroupVersionKind(schema.GroupVersionKind{}) 64 | if crd.Labels == nil { 65 | crd.Labels = make(map[string]string) 66 | } 67 | crd.Labels[constants.CreatedByLabel] = constants.CreatedBy 68 | crds = append(crds, crd) 69 | } 70 | 71 | return crds, nil 72 | } 73 | 74 | func LoadCRDs() ([]*apiextensionsv1.CustomResourceDefinition, error) { 75 | crds := make([]*apiextensionsv1.CustomResourceDefinition, 0) 76 | dirEntrys, err := fcrds.ReadDir("crds") 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | for _, entry := range dirEntrys { 82 | if entry.IsDir() { 83 | continue 84 | } 85 | 86 | rawByte, err := fcrds.ReadFile(fmt.Sprintf("crds/%s", entry.Name())) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | tmp, err := load(bytes.NewReader(rawByte)) 92 | if err != nil { 93 | return crds, err 94 | } 95 | 96 | crds = append(crds, tmp...) 97 | } 98 | 99 | return crds, nil 100 | } 101 | -------------------------------------------------------------------------------- /pkg/static/crds/devops.fake.io_clustercredentials.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.5.0 8 | creationTimestamp: null 9 | name: clustercredentials.devops.fake.io 10 | spec: 11 | group: devops.fake.io 12 | names: 13 | kind: ClusterCredential 14 | listKind: ClusterCredentialList 15 | plural: clustercredentials 16 | singular: clustercredential 17 | scope: Namespaced 18 | versions: 19 | - name: v1 20 | schema: 21 | openAPIV3Schema: 22 | description: ClusterCredential records the credential information needed to access the cluster. 23 | properties: 24 | apiVersion: 25 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 26 | type: string 27 | bootstrapToken: 28 | description: For kubeadm init or join 29 | type: string 30 | caCert: 31 | description: For connect the cluster 32 | format: byte 33 | type: string 34 | caKey: 35 | format: byte 36 | type: string 37 | certificateKey: 38 | description: For kubeadm init or join 39 | type: string 40 | certsBinaryData: 41 | additionalProperties: 42 | format: byte 43 | type: string 44 | type: object 45 | clientCert: 46 | description: For kube-apiserver X509 auth 47 | format: byte 48 | type: string 49 | clientKey: 50 | description: For kube-apiserver X509 auth 51 | format: byte 52 | type: string 53 | clusterName: 54 | type: string 55 | etcdAPIClientCert: 56 | format: byte 57 | type: string 58 | etcdAPIClientKey: 59 | format: byte 60 | type: string 61 | etcdCACert: 62 | description: For TKE in global reuse 63 | format: byte 64 | type: string 65 | etcdCAKey: 66 | format: byte 67 | type: string 68 | extData: 69 | additionalProperties: 70 | type: string 71 | type: object 72 | kind: 73 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 74 | type: string 75 | kubeData: 76 | additionalProperties: 77 | type: string 78 | type: object 79 | manifestsData: 80 | additionalProperties: 81 | type: string 82 | type: object 83 | metadata: 84 | type: object 85 | tenantID: 86 | type: string 87 | token: 88 | description: For kube-apiserver token auth 89 | type: string 90 | required: 91 | - clusterName 92 | - tenantID 93 | type: object 94 | served: true 95 | storage: true 96 | status: 97 | acceptedNames: 98 | kind: "" 99 | plural: "" 100 | conditions: [] 101 | storedVersions: [] 102 | -------------------------------------------------------------------------------- /pkg/static/crds/workload.fake.io_addons.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.5.0 8 | creationTimestamp: null 9 | name: addons.workload.fake.io 10 | spec: 11 | group: workload.fake.io 12 | names: 13 | kind: Addons 14 | listKind: AddonsList 15 | plural: addons 16 | singular: addons 17 | scope: Namespaced 18 | versions: 19 | - additionalPrinterColumns: 20 | - description: The Addons phase. 21 | jsonPath: .status.phase 22 | name: PHASE 23 | type: string 24 | - description: 'CreationTimestamp is a timestamp representing the server time when this object was created. ' 25 | jsonPath: .metadata.creationTimestamp 26 | name: AGE 27 | type: date 28 | name: v1 29 | schema: 30 | openAPIV3Schema: 31 | description: Machine is the Schema for the Machine API 32 | properties: 33 | apiVersion: 34 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 35 | type: string 36 | kind: 37 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 38 | type: string 39 | metadata: 40 | type: object 41 | spec: 42 | description: AddonsSpec is a description of Addons. 43 | properties: 44 | foo: 45 | type: string 46 | type: object 47 | status: 48 | description: AddonsStatus represents information about the status of an Addons. 49 | properties: 50 | foo: 51 | type: string 52 | phase: 53 | type: string 54 | type: object 55 | type: object 56 | served: true 57 | storage: true 58 | subresources: 59 | status: {} 60 | status: 61 | acceptedNames: 62 | kind: "" 63 | plural: "" 64 | conditions: [] 65 | storedVersions: [] 66 | -------------------------------------------------------------------------------- /pkg/util/allocator/interfaces.go: -------------------------------------------------------------------------------- 1 | package allocator 2 | 3 | // Interface manages the allocation of items out of a range. Interface 4 | // should be threadsafe. 5 | type Interface interface { 6 | Allocate(int) (bool, error) 7 | AllocateNext() (int, bool, error) 8 | Release(int) error 9 | ForEach(func(int)) 10 | 11 | // For testing 12 | Has(int) bool 13 | 14 | // For testing 15 | Free() int 16 | } 17 | 18 | // Snapshottable is an Interface that can be snapshotted and restored. Snapshottable 19 | // should be threadsafe. 20 | type Snapshottable interface { 21 | Interface 22 | Snapshot() (string, []byte) 23 | Restore(string, []byte) error 24 | } 25 | 26 | type Factory func(max int, rangeSpec string) Interface 27 | -------------------------------------------------------------------------------- /pkg/util/allocator/utils.go: -------------------------------------------------------------------------------- 1 | package allocator 2 | 3 | import "math/big" 4 | 5 | // countBits returns the number of set bits in n 6 | func countBits(n *big.Int) int { 7 | var count = 0 8 | for _, b := range n.Bytes() { 9 | count += int(bitCounts[b]) 10 | } 11 | return count 12 | } 13 | 14 | // bitCounts is all of the bits counted for each number between 0-255 15 | var bitCounts = []int8{ 16 | 0, 1, 1, 2, 1, 2, 2, 3, 17 | 1, 2, 2, 3, 2, 3, 3, 4, 18 | 1, 2, 2, 3, 2, 3, 3, 4, 19 | 2, 3, 3, 4, 3, 4, 4, 5, 20 | 1, 2, 2, 3, 2, 3, 3, 4, 21 | 2, 3, 3, 4, 3, 4, 4, 5, 22 | 2, 3, 3, 4, 3, 4, 4, 5, 23 | 3, 4, 4, 5, 4, 5, 5, 6, 24 | 1, 2, 2, 3, 2, 3, 3, 4, 25 | 2, 3, 3, 4, 3, 4, 4, 5, 26 | 2, 3, 3, 4, 3, 4, 4, 5, 27 | 3, 4, 4, 5, 4, 5, 5, 6, 28 | 2, 3, 3, 4, 3, 4, 4, 5, 29 | 3, 4, 4, 5, 4, 5, 5, 6, 30 | 3, 4, 4, 5, 4, 5, 5, 6, 31 | 4, 5, 5, 6, 5, 6, 6, 7, 32 | 1, 2, 2, 3, 2, 3, 3, 4, 33 | 2, 3, 3, 4, 3, 4, 4, 5, 34 | 2, 3, 3, 4, 3, 4, 4, 5, 35 | 3, 4, 4, 5, 4, 5, 5, 6, 36 | 2, 3, 3, 4, 3, 4, 4, 5, 37 | 3, 4, 4, 5, 4, 5, 5, 6, 38 | 3, 4, 4, 5, 4, 5, 5, 6, 39 | 4, 5, 5, 6, 5, 6, 6, 7, 40 | 2, 3, 3, 4, 3, 4, 4, 5, 41 | 3, 4, 4, 5, 4, 5, 5, 6, 42 | 3, 4, 4, 5, 4, 5, 5, 6, 43 | 4, 5, 5, 6, 5, 6, 6, 7, 44 | 3, 4, 4, 5, 4, 5, 5, 6, 45 | 4, 5, 5, 6, 5, 6, 6, 7, 46 | 4, 5, 5, 6, 5, 6, 6, 7, 47 | 5, 6, 6, 7, 6, 7, 7, 8, 48 | } 49 | -------------------------------------------------------------------------------- /pkg/util/apiclient/version.go: -------------------------------------------------------------------------------- 1 | package apiclient 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Masterminds/semver" 7 | "k8s.io/client-go/kubernetes" 8 | ) 9 | 10 | func ClusterVersionIsBefore19(client kubernetes.Interface) bool { 11 | result, err := CheckClusterVersion(client, "< 1.9") 12 | if err != nil { 13 | return false 14 | } 15 | 16 | return result 17 | } 18 | 19 | func CheckClusterVersion(client kubernetes.Interface, versionConstraint string) (bool, error) { 20 | version, err := GetClusterVersion(client) 21 | if err != nil { 22 | return false, err 23 | } 24 | 25 | return CheckVersion(version, versionConstraint) 26 | } 27 | 28 | func GetClusterVersion(client kubernetes.Interface) (string, error) { 29 | version, err := client.Discovery().ServerVersion() 30 | if err != nil { 31 | return "", err 32 | } 33 | return fmt.Sprintf("%v.%v", version.Major, version.Minor), nil 34 | } 35 | 36 | func CheckVersion(version string, versionConstraint string) (bool, error) { 37 | c, err := semver.NewConstraint(versionConstraint) 38 | if err != nil { 39 | return false, err 40 | } 41 | v, err := semver.NewVersion(version) 42 | if err != nil { 43 | return false, err 44 | } 45 | 46 | return c.Check(v), nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/util/hash/hash.go: -------------------------------------------------------------------------------- 1 | package hash 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "hash" 7 | "io/ioutil" 8 | "os" 9 | ) 10 | 11 | func Sha256WithFile(filename string) (string, error) { 12 | h := sha256.New() 13 | return SumWithFile(h, filename) 14 | } 15 | 16 | func SumWithFile(h hash.Hash, filename string) (string, error) { 17 | f, err := os.Open(filename) 18 | if err != nil { 19 | return "", err 20 | } 21 | defer f.Close() 22 | 23 | data, err := ioutil.ReadAll(f) 24 | if err != nil { 25 | return "", err 26 | } 27 | 28 | return Sum(h, data), nil 29 | } 30 | 31 | func Sum(h hash.Hash, data []byte) string { 32 | return hex.EncodeToString(h.Sum(data)) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/util/hosts/hosts.go: -------------------------------------------------------------------------------- 1 | package hosts 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | ) 7 | 8 | const ( 9 | linuxHostfile = "/etc/hosts" 10 | ) 11 | 12 | // Hostser for hosts 13 | type Hostser interface { 14 | Data() ([]byte, error) 15 | Set(ip string) error 16 | } 17 | 18 | func hostFile() string { 19 | return linuxHostfile 20 | } 21 | 22 | func setHosts(data []byte, host, ip string) ([]byte, error) { 23 | item := fmt.Sprintf("%s %s", ip, host) 24 | var re = regexp.MustCompile(fmt.Sprintf(".* %s", host)) 25 | var newData string 26 | if re.Match(data) { 27 | newData = re.ReplaceAllString(string(data), item) 28 | } else { 29 | newData = fmt.Sprintf("%s\n%s\n", data, item) 30 | } 31 | 32 | return []byte(newData), nil 33 | } 34 | -------------------------------------------------------------------------------- /pkg/util/hosts/local.go: -------------------------------------------------------------------------------- 1 | package hosts 2 | 3 | import "io/ioutil" 4 | 5 | // LocalHosts for local hosts 6 | type LocalHosts struct { 7 | Host string 8 | File string 9 | } 10 | 11 | // Set sets hosts 12 | func (h *LocalHosts) Set(ip string) error { 13 | if h.File == "" { 14 | h.File = hostFile() 15 | } 16 | data, err := ioutil.ReadFile(h.File) 17 | if err != nil { 18 | return err 19 | } 20 | data, err = setHosts(data, h.Host, ip) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | return ioutil.WriteFile(h.File, data, 0644) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/util/hosts/remote.go: -------------------------------------------------------------------------------- 1 | package hosts 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/wtxue/kok-operator/pkg/util/ssh" 7 | ) 8 | 9 | // RemoteHosts for remote hosts 10 | type RemoteHosts struct { 11 | Host string 12 | SSH ssh.Interface 13 | } 14 | 15 | // Data return hosts data 16 | func (h *RemoteHosts) Data() ([]byte, error) { 17 | return h.SSH.ReadFile(linuxHostfile) 18 | } 19 | 20 | // Set sets hosts 21 | func (h *RemoteHosts) Set(ip string) error { 22 | data, err := h.Data() 23 | if err != nil { 24 | return err 25 | } 26 | data, err = setHosts(data, h.Host, ip) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | return h.SSH.WriteFile(bytes.NewReader(data), linuxHostfile) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/util/kubeconfig/kubeconfig.go: -------------------------------------------------------------------------------- 1 | package kubeconfig 2 | 3 | import ( 4 | "fmt" 5 | 6 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 7 | ) 8 | 9 | // CreateBasic creates a basic, general KubeConfig object that then can be extended 10 | func CreateBasic(serverURL, clusterName, userName string, caCert []byte) *clientcmdapi.Config { 11 | // Use the cluster and the username as the context name 12 | contextName := fmt.Sprintf("%s@%s", userName, clusterName) 13 | 14 | return &clientcmdapi.Config{ 15 | Clusters: map[string]*clientcmdapi.Cluster{ 16 | clusterName: { 17 | Server: serverURL, 18 | CertificateAuthorityData: caCert, 19 | }, 20 | }, 21 | Contexts: map[string]*clientcmdapi.Context{ 22 | contextName: { 23 | Cluster: clusterName, 24 | AuthInfo: userName, 25 | }, 26 | }, 27 | AuthInfos: map[string]*clientcmdapi.AuthInfo{}, 28 | CurrentContext: contextName, 29 | } 30 | } 31 | 32 | // CreateWithCerts creates a KubeConfig object with access to the API server with client certificates 33 | func CreateWithCerts(serverURL, clusterName, userName string, caCert []byte, clientKey []byte, clientCert []byte) *clientcmdapi.Config { 34 | config := CreateBasic(serverURL, clusterName, userName, caCert) 35 | config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ 36 | ClientKeyData: clientKey, 37 | ClientCertificateData: clientCert, 38 | } 39 | return config 40 | } 41 | 42 | // CreateWithToken creates a KubeConfig object with access to the API server with a token 43 | func CreateWithToken(serverURL, clusterName, userName string, caCert []byte, token string) *clientcmdapi.Config { 44 | config := CreateBasic(serverURL, clusterName, userName, caCert) 45 | config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ 46 | Token: token, 47 | } 48 | return config 49 | } 50 | -------------------------------------------------------------------------------- /pkg/util/pointer/pointer.go: -------------------------------------------------------------------------------- 1 | package pointer 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | /* 8 | Order as in spec: 9 | bool byte complex64 complex128 error float32 float64 10 | int int8 int16 int32 int64 rune string 11 | uint uint8 uint16 uint32 uint64 uintptr 12 | time.Duration time.Time 13 | */ 14 | 15 | // ToBool returns a pointer to the passed bool value. 16 | func ToBool(b bool) *bool { 17 | return &b 18 | } 19 | 20 | // ToByte returns a pointer to the passed byte value. 21 | func ToByte(b byte) *byte { 22 | return &b 23 | } 24 | 25 | // ToComplex64 returns a pointer to the passed complex64 value. 26 | func ToComplex64(c complex64) *complex64 { 27 | return &c 28 | } 29 | 30 | // ToComplex128 returns a pointer to the passed complex128 value. 31 | func ToComplex128(c complex128) *complex128 { 32 | return &c 33 | } 34 | 35 | // ToError returns a pointer to the passed error value. 36 | func ToError(e error) *error { 37 | return &e 38 | } 39 | 40 | // ToFloat32 returns a pointer to the passed float32 value. 41 | func ToFloat32(f float32) *float32 { 42 | return &f 43 | } 44 | 45 | // ToFloat64 returns a pointer to the passed float64 value. 46 | func ToFloat64(f float64) *float64 { 47 | return &f 48 | } 49 | 50 | // ToInt returns a pointer to the passed int value. 51 | func ToInt(i int) *int { 52 | return &i 53 | } 54 | 55 | // ToInt8 returns a pointer to the passed int8 value. 56 | func ToInt8(i int8) *int8 { 57 | return &i 58 | } 59 | 60 | // ToInt16 returns a pointer to the passed int16 value. 61 | func ToInt16(i int16) *int16 { 62 | return &i 63 | } 64 | 65 | // ToInt32 returns a pointer to the passed int32 value. 66 | func ToInt32(i int32) *int32 { 67 | return &i 68 | } 69 | 70 | // ToInt64 returns a pointer to the passed int64 value. 71 | func ToInt64(i int64) *int64 { 72 | return &i 73 | } 74 | 75 | // ToRune returns a pointer to the passed rune value. 76 | func ToRune(r rune) *rune { 77 | return &r 78 | } 79 | 80 | // ToString returns a pointer to the passed string value. 81 | func ToString(s string) *string { 82 | return &s 83 | } 84 | 85 | // ToUint returns a pointer to the passed uint value. 86 | func ToUint(u uint) *uint { 87 | return &u 88 | } 89 | 90 | // ToUint8 returns a pointer to the passed uint8 value. 91 | func ToUint8(u uint8) *uint8 { 92 | return &u 93 | } 94 | 95 | // ToUint16 returns a pointer to the passed uint16 value. 96 | func ToUint16(u uint16) *uint16 { 97 | return &u 98 | } 99 | 100 | // ToUint32 returns a pointer to the passed uint32 value. 101 | func ToUint32(u uint32) *uint32 { 102 | return &u 103 | } 104 | 105 | // ToUint64 returns a pointer to the passed uint64 value. 106 | func ToUint64(u uint64) *uint64 { 107 | return &u 108 | } 109 | 110 | // ToUintptr returns a pointer to the passed uintptr value. 111 | func ToUintptr(u uintptr) *uintptr { 112 | return &u 113 | } 114 | 115 | // ToDuration returns a pointer to the passed time.Duration value. 116 | func ToDuration(d time.Duration) *time.Duration { 117 | return &d 118 | } 119 | 120 | // ToTime returns a pointer to the passed time.Time value. 121 | func ToTime(t time.Time) *time.Time { 122 | return &t 123 | } 124 | -------------------------------------------------------------------------------- /pkg/util/ssh/helpers.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | // GetNetworkInterface return network interface name by ip 12 | func GetNetworkInterface(s Interface, ip string) string { 13 | stdout, _, _, _ := s.Execf("ip a | grep '%s' |awk '{print $NF}'", ip) 14 | 15 | return stdout 16 | } 17 | 18 | // Timestamp returns target node timestamp. 19 | func Timestamp(s Interface) (int, error) { 20 | stdout, err := s.CombinedOutput("date +%s") 21 | if err != nil { 22 | return 0, err 23 | } 24 | 25 | return strconv.Atoi(strings.TrimSpace(string(stdout))) 26 | } 27 | 28 | func BackupFile(s Interface, file string) (string, error) { 29 | backup := path.Join(fmt.Sprintf("%s-%s", file, time.Now().Format("20060102150405"))) 30 | cmd := fmt.Sprintf("mv %s %s", file, backup) 31 | _, err := s.CombinedOutput(cmd) 32 | if err != nil { 33 | return "", fmt.Errorf("backup %q error: %w", file, err) 34 | } 35 | 36 | return backup, nil 37 | } 38 | 39 | func RestoreFile(s Interface, file string) error { 40 | i := strings.LastIndex(file, "-") 41 | if i <= 0 { 42 | return fmt.Errorf("invalid file name %q", file) 43 | } 44 | cmd := fmt.Sprintf("mv %s %s", file, file[0:i]) 45 | _, err := s.CombinedOutput(cmd) 46 | if err != nil { 47 | return fmt.Errorf("restore %q error: %w", file, err) 48 | } 49 | 50 | return nil 51 | } 52 | 53 | // MemoryCapacity returns the machine's total memory from /proc/meminfo. 54 | // Returns the total memory capacity as an uint64 (number of bytes). 55 | func MemoryCapacity(s Interface) (uint64, error) { 56 | stdout, err := s.CombinedOutput(`grep 'MemTotal:' /proc/meminfo | grep -oP '\d+'`) 57 | if err != nil { 58 | return 0, err 59 | } 60 | 61 | memInKB, err := strconv.ParseUint(strings.TrimSpace(string(stdout)), 10, 64) 62 | if err != nil { 63 | return 0, err 64 | } 65 | 66 | return memInKB * 1024, err 67 | } 68 | 69 | // NumCPU returns the number of logical CPUs. 70 | func NumCPU(s Interface) (int, error) { 71 | stdout, err := s.CombinedOutput(`nproc --all`) 72 | if err != nil { 73 | return 0, err 74 | } 75 | 76 | cpu, err := strconv.Atoi(strings.TrimSpace(string(stdout))) 77 | if err != nil { 78 | return 0, err 79 | } 80 | 81 | return cpu, nil 82 | } 83 | 84 | // DiskAvail returns available disk space in GiB. 85 | func DiskAvail(s Interface, path string) (int, error) { 86 | cmd := fmt.Sprintf(`df -BG %s | tail -1 | awk '{print $4}' | grep -oP '\d+'`, path) 87 | stdout, err := s.CombinedOutput(cmd) 88 | if err != nil { 89 | return 0, err 90 | } 91 | 92 | disk, err := strconv.Atoi(strings.TrimSpace(string(stdout))) 93 | if err != nil { 94 | return 0, err 95 | } 96 | 97 | return disk, nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/util/supervisor/systemd.go: -------------------------------------------------------------------------------- 1 | package supervisor 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "path" 7 | 8 | "github.com/wtxue/kok-operator/pkg/util/ssh" 9 | ) 10 | 11 | const ( 12 | DefaultSystemdUnitFilePath = "/etc/systemd/system" 13 | ) 14 | 15 | type SystemdSupervisor struct { 16 | Name string 17 | SSH ssh.Interface 18 | } 19 | 20 | func (s *SystemdSupervisor) Deploy(data io.Reader) error { 21 | unitFilePath := path.Join(DefaultSystemdUnitFilePath, fmt.Sprintf("%s.service", s.Name)) 22 | if err := s.SSH.WriteFile(data, unitFilePath); err != nil { 23 | return err 24 | } 25 | 26 | cmd := fmt.Sprintf("systemctl -f enable %s", unitFilePath) 27 | if _, stderr, exit, err := s.SSH.Execf(cmd); err != nil || exit != 0 { 28 | return fmt.Errorf("exec %q failed:exit %d:stderr %s:error %s", cmd, exit, stderr, err) 29 | } 30 | 31 | cmd = "systemctl daemon-reload" 32 | if _, stderr, exit, err := s.SSH.Execf(cmd); err != nil || exit != 0 { 33 | return fmt.Errorf("exec %q failed:exit %d:stderr %s:error %s", cmd, exit, stderr, err) 34 | } 35 | 36 | return nil 37 | } 38 | 39 | func (s *SystemdSupervisor) Start() error { 40 | unitName := fmt.Sprintf("%s.service", s.Name) 41 | 42 | cmd := fmt.Sprintf("systemctl restart %s", unitName) 43 | if _, stderr, exit, err := s.SSH.Execf(cmd); err != nil || exit != 0 { 44 | cmd = fmt.Sprintf("journalctl --unit %s -n10 --no-pager", unitName) 45 | jStdout, _, jExit, jErr := s.SSH.Execf(cmd) 46 | if jErr != nil || jExit != 0 { 47 | return fmt.Errorf("exec %q:error %s", cmd, err) 48 | } 49 | fmt.Printf("log:\n%s", jStdout) 50 | 51 | return fmt.Errorf("Exec %s failed:exit %d:stderr %s:error %s:log:\n%s", cmd, exit, stderr, err, jStdout) 52 | } 53 | 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/util/template/template.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "text/template" 7 | 8 | "github.com/Masterminds/sprig" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // ParseFile validates and parses passed as argument template file 13 | func ParseFile(filename string, obj interface{}) ([]byte, error) { 14 | data, err := ioutil.ReadFile(filename) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | return ParseString(string(data), obj) 20 | } 21 | 22 | // ParseString validates and parses passed as argument template 23 | func ParseString(strtmpl string, obj interface{}) ([]byte, error) { 24 | var buf bytes.Buffer 25 | tmpl, err := template.New("template").Funcs(sprig.TxtFuncMap()).Parse(strtmpl) 26 | if err != nil { 27 | return nil, errors.Wrap(err, "error when parsing template") 28 | } 29 | err = tmpl.Execute(&buf, obj) 30 | if err != nil { 31 | return nil, errors.Wrap(err, "error when executing template") 32 | } 33 | return buf.Bytes(), nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/util/validation/kubernetes.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | 8 | "github.com/thoas/go-funk" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/apimachinery/pkg/util/validation/field" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/rest" 13 | ) 14 | 15 | // ValidateEnum validates a given enum. 16 | // nil or nil pointer is valid. 17 | // zero value is invalid. 18 | func ValidateEnum(value interface{}, fldPath *field.Path, values interface{}) field.ErrorList { 19 | allErrs := field.ErrorList{} 20 | 21 | if value == nil { 22 | return allErrs 23 | } 24 | 25 | validValuesString := funk.Map(values, func(i interface{}) string { 26 | return fmt.Sprintf("%v", i) 27 | }).([]string) 28 | 29 | v := reflect.ValueOf(value) 30 | if v.Kind() == reflect.Ptr { 31 | if reflect.ValueOf(value).IsNil() { 32 | return allErrs 33 | } 34 | } else { 35 | if v.IsZero() { 36 | allErrs = append(allErrs, field.Required(fldPath, fmt.Sprintf("valid values: %v", validValuesString))) 37 | return allErrs 38 | } 39 | } 40 | 41 | if !funk.Contains(values, value) { 42 | allErrs = append(allErrs, field.NotSupported(fldPath, value, validValuesString)) 43 | } 44 | 45 | return allErrs 46 | } 47 | 48 | // ValidateRESTConfig validates a given rest.Config. 49 | func ValidateRESTConfig(ctx context.Context, config *rest.Config) error { 50 | clientset, err := kubernetes.NewForConfig(config) 51 | if err != nil { 52 | return err 53 | } 54 | _, err = clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /pkg/util/validation/name.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | ) 7 | 8 | const dns1123NameMaxLength int = 32 9 | const dns1123NameMinLength int = 3 10 | const displayNameMaxLength int = 255 11 | 12 | var dns1123NameFmt = "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" 13 | var emailFmt = `^[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,4}$` 14 | var phoneNumberFmt = `^1[3|4|5|7|8][0-9]{9}$` 15 | 16 | var dns1123NameRegexp = regexp.MustCompile(dns1123NameFmt) 17 | var emailRegexp = regexp.MustCompile(emailFmt) 18 | var phoneNumberRegexp = regexp.MustCompile(phoneNumberFmt) 19 | 20 | // IsDNS1123Name tests for a string that conforms to the definition of a name in 21 | // DNS (RFC 1123). 22 | func IsDNS1123Name(value string) error { 23 | if value == "" { 24 | return fmt.Errorf("must be specified") 25 | } 26 | if len(value) < dns1123NameMinLength || len(value) > dns1123NameMaxLength { 27 | return fmt.Errorf("length must be greater than %d and less than %d", dns1123NameMinLength, dns1123NameMaxLength) 28 | } 29 | if !dns1123NameRegexp.MatchString(value) { 30 | return fmt.Errorf("must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character") 31 | } 32 | return nil 33 | } 34 | 35 | // IsDisplayName test whether the given value meets the specification of the 36 | // display name. 37 | func IsDisplayName(value string) error { 38 | if value == "" { 39 | return fmt.Errorf("must be specified") 40 | } 41 | 42 | if len(value) > displayNameMaxLength { 43 | return fmt.Errorf("length must be less than %d", displayNameMaxLength) 44 | } 45 | return nil 46 | } 47 | 48 | // IsEmail test whether the given value meets the specification of the email. 49 | func IsEmail(value string) error { 50 | if value == "" { 51 | return fmt.Errorf("must be specified") 52 | } 53 | 54 | if !emailRegexp.MatchString(value) { 55 | return fmt.Errorf("email is not valid format, must satisfy regex %s, examples: 123@abc.com ", emailRegexp) 56 | } 57 | 58 | return nil 59 | } 60 | 61 | // IsPhoneNumber test whether the given value meets the specification of the phone number. 62 | func IsPhoneNumber(value string) error { 63 | if value == "" { 64 | return fmt.Errorf("must be specified") 65 | } 66 | 67 | if !phoneNumberRegexp.MatchString(value) { 68 | return fmt.Errorf("phoneNumer is not valid format, must satisfy regex %s, examples: 13611111111", emailRegexp) 69 | } 70 | 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /pkg/util/validation/url.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | ) 7 | 8 | var urlRegex = regexp.MustCompile("(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]") 9 | 10 | // IsURL test whether the given value is a valid URL address. 11 | func IsURL(value string) error { 12 | if !urlRegex.MatchString(value) { 13 | return fmt.Errorf("not a valid URL address") 14 | } 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /pkg/util/validation/validation.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/pkg/errors" 11 | "github.com/wtxue/kok-operator/pkg/util/ipallocator" 12 | ) 13 | 14 | // IsHTTPSReachle tests that https://host:port is reachble in timeout. 15 | func IsHTTPSReachle(host string, port int32, timeout time.Duration) error { 16 | client := &http.Client{ 17 | Transport: &http.Transport{ 18 | Proxy: http.ProxyFromEnvironment, 19 | DialContext: (&net.Dialer{ 20 | Timeout: timeout, 21 | }).DialContext, 22 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 23 | }, 24 | } 25 | 26 | url := fmt.Sprintf("https://%s:%d", host, port) 27 | request, err := http.NewRequest(http.MethodGet, url, nil) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | _, err = client.Do(request) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | return nil 38 | } 39 | 40 | // IsSubNetOverlapped test if two subnets are overlapped 41 | func IsSubNetOverlapped(net1, net2 *net.IPNet) error { 42 | if net1 == nil || net2 == nil { 43 | return nil 44 | } 45 | net1FirstIP, _ := ipallocator.GetFirstIP(net1) 46 | net1LastIP, _ := ipallocator.GetLastIP(net1) 47 | 48 | net2FirstIP, _ := ipallocator.GetFirstIP(net2) 49 | net2LastIP, _ := ipallocator.GetLastIP(net2) 50 | 51 | if net1.Contains(net2FirstIP) || net1.Contains(net2LastIP) || 52 | net2.Contains(net1FirstIP) || net2.Contains(net1LastIP) { 53 | return errors.Errorf("subnet %v and %v are overlapped", net1, net2) 54 | } 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | ) 9 | 10 | var ( 11 | // Release returns the release version 12 | Release = "UNKNOWN" 13 | // Commit returns the short sha from git 14 | Commit = "UNKNOWN" 15 | // BuildDate is the build date 16 | BuildDate = "" 17 | ) 18 | 19 | type Version struct { 20 | GitCommit string 21 | BuildDate string 22 | Release string 23 | GoVersion string 24 | Compiler string 25 | Platform string 26 | } 27 | 28 | func (v Version) String() string { 29 | return fmt.Sprintf("%s/%s (%s/%s) Date/%s GitCommit/%s", 30 | filepath.Base(os.Args[0]), v.Release, runtime.GOOS, runtime.GOARCH, v.BuildDate, v.GitCommit) 31 | } 32 | 33 | // GetVersion returns version 34 | func GetVersion() Version { 35 | return Version{ 36 | GitCommit: Commit, 37 | BuildDate: BuildDate, 38 | Release: Release, 39 | GoVersion: runtime.Version(), 40 | Compiler: runtime.Compiler, 41 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tools/image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -xeuo pipefail 4 | 5 | K8s_Version=${1:-"v1.24.4"} 6 | Etcd_Version=${2:-"3.5.3-0"} 7 | CoreDns_Version=${3:-"v1.8.6"} 8 | DstImagePrefix=${4:-"docker.io/wtxue"} 9 | Pause_Version=${5:-"3.7"} 10 | 11 | docker pull k8s.gcr.io/kube-apiserver:${K8s_Version} && \ 12 | docker tag k8s.gcr.io/kube-apiserver:${K8s_Version} ${DstImagePrefix}/kube-apiserver:${K8s_Version} && \ 13 | docker push ${DstImagePrefix}/kube-apiserver:${K8s_Version} 14 | 15 | docker pull k8s.gcr.io/kube-controller-manager:${K8s_Version} && \ 16 | docker tag k8s.gcr.io/kube-controller-manager:${K8s_Version} ${DstImagePrefix}/kube-controller-manager:${K8s_Version} && \ 17 | docker push ${DstImagePrefix}/kube-controller-manager:${K8s_Version} 18 | 19 | docker pull k8s.gcr.io/kube-scheduler:${K8s_Version} && \ 20 | docker tag k8s.gcr.io/kube-scheduler:${K8s_Version} ${DstImagePrefix}/kube-scheduler:${K8s_Version} && \ 21 | docker push ${DstImagePrefix}/kube-scheduler:${K8s_Version} 22 | 23 | docker pull k8s.gcr.io/kube-proxy:${K8s_Version} && \ 24 | docker tag k8s.gcr.io/kube-proxy:${K8s_Version} ${DstImagePrefix}/kube-proxy:${K8s_Version} && \ 25 | docker push ${DstImagePrefix}/kube-proxy:${K8s_Version} 26 | 27 | docker pull k8s.gcr.io/pause:${Pause_Version} && \ 28 | docker tag k8s.gcr.io/pause:${Pause_Version} ${DstImagePrefix}/pause:${Pause_Version} && \ 29 | docker push ${DstImagePrefix}/pause:${Pause_Version} 30 | 31 | docker pull k8s.gcr.io/etcd:${Etcd_Version} && \ 32 | docker tag k8s.gcr.io/etcd:${Etcd_Version} ${DstImagePrefix}/etcd:${Etcd_Version} && \ 33 | docker push ${DstImagePrefix}/etcd:${Etcd_Version} 34 | 35 | docker pull k8s.gcr.io/coredns/coredns:${CoreDns_Version} && \ 36 | docker tag k8s.gcr.io/coredns/coredns:${CoreDns_Version} ${DstImagePrefix}/coredns:${CoreDns_Version} && \ 37 | docker push ${DstImagePrefix}/coredns:${CoreDns_Version} 38 | 39 | 40 | -------------------------------------------------------------------------------- /tools/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e # exit immediately on error 4 | set -x # display all commands 5 | 6 | PACKAGE_DIR="../k8s" 7 | 8 | SysOS=`uname -s` 9 | TargetOS="darwin" 10 | Version="2.3.1" 11 | if [ $SysOS == "Darwin" ];then 12 | TargetOS="darwin" 13 | else 14 | TargetOS="linux" 15 | fi 16 | 17 | if [ ! -f ${PACKAGE_DIR}/bin/kube-apiserver ]; then 18 | if [ ! -f kubebuilder_${Version}_${TargetOS}_amd64.tar.gz ]; then 19 | wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${Version}/kubebuilder_${Version}_${TargetOS}_amd64.tar.gz 20 | fi 21 | 22 | tar -xf kubebuilder_${Version}_${TargetOS}_amd64.tar.gz 23 | mkdir -p PACKAGE_BIN 24 | cp -rf kubebuilder_${Version}_${TargetOS}_amd64/bin ${PACKAGE_DIR}/ 25 | rm -rf kubebuilder_${Version}_${TargetOS}_amd64.tar.gz kubebuilder_${Version}_${TargetOS}_amd64 26 | fi 27 | 28 | 29 | echo "all done." 30 | -------------------------------------------------------------------------------- /tools/kubeadmin-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeadm.k8s.io/v1beta2 2 | bootstrapTokens: 3 | - groups: 4 | - system:bootstrappers:kubeadm:default-node-token 5 | token: abcdef.0123456789abcdef 6 | ttl: 24h0m0s 7 | usages: 8 | - signing 9 | - authentication 10 | kind: InitConfiguration 11 | localAPIEndpoint: 12 | advertiseAddress: 192.168.99.70 13 | bindPort: 6443 14 | nodeRegistration: 15 | criSocket: /var/run/dockershim.sock 16 | name: k8s-node1 17 | taints: 18 | - effect: NoSchedule 19 | key: node-role.kubernetes.io/master 20 | --- 21 | apiServer: 22 | certSANs: 23 | - 192.168.99.70 24 | - 192.168.99.71 25 | - 192.168.99.72 26 | - 192.168.99.253 27 | - k8s-master1 28 | - vip.local.xx.com 29 | timeoutForControlPlane: 4m0s 30 | apiVersion: kubeadm.k8s.io/v1beta2 31 | certificatesDir: /etc/kubernetes/pki 32 | clusterName: kubernetes 33 | controllerManager: {} 34 | dns: 35 | type: CoreDNS 36 | etcd: 37 | local: 38 | dataDir: /var/lib/etcd 39 | imageRepository: k8s.gcr.io 40 | kind: ClusterConfiguration 41 | kubernetesVersion: v1.18.0 42 | networking: 43 | dnsDomain: cluster.local 44 | serviceSubnet: 10.96.0.0/12 45 | scheduler: {} -------------------------------------------------------------------------------- /tools/new-init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | etcd-v3.4.9-darwin-amd64.zip 3 | 4 | set -e # exit immediately on error 5 | set -x # display all commands 6 | 7 | PACKAGE_DIR="../k8s" 8 | 9 | sysOS=`uname -s` 10 | TargetOS="darwin" 11 | EtcdVersion="3.4.9" 12 | K8sVersion="1.18.3" 13 | if [ $sysOS == "Darwin" ];then 14 | TargetOS="darwin" 15 | else 16 | TargetOS="linux" 17 | fi 18 | 19 | if [ ! -f ${PACKAGE_DIR}/bin/etcd ]; then 20 | if [ ! -f etcd-v${EtcdVersion}-${TargetOS}-amd64.zip ]; then 21 | wget https://github.com/etcd-io/etcd/releases/download/v${EtcdVersion}/etcd-v${EtcdVersion}-${TargetOS}-amd64.zip 22 | fi 23 | 24 | tar -xf etcd-v${EtcdVersion}-${TargetOS}-amd64.zip 25 | mkdir -p ${PACKAGE_DIR}/bin 26 | cp -f etcd-v${EtcdVersion}-${TargetOS}-amd64/etcd* ${PACKAGE_DIR}/bin/ 27 | rm -rf etcd-v${EtcdVersion}-${TargetOS}-amd64.zip etcd-v${EtcdVersion}-${TargetOS}-amd64 28 | fi 29 | 30 | if [ ! -f ${PACKAGE_DIR}/bin/kube-apiserver ]; then 31 | if [ ! -f kubernetes-server-linux-amd64.tar.gz ]; then 32 | wget https://dl.k8s.io/v${K8sVersion}/kubernetes-server-linux-amd64.tar.gz 33 | fi 34 | 35 | tar -xf kubernetes-server-linux-amd64.tar.gz 36 | # mkdir -p ${PACKAGE_DIR}/bin 37 | # cp -f etcd-v${EtcdVersion}-${TargetOS}-amd64/etcd* ${PACKAGE_DIR}/bin/ 38 | # rm -rf etcd-v${EtcdVersion}-${TargetOS}-amd64.zip etcd-v${EtcdVersion}-${TargetOS}-amd64 39 | fi 40 | 41 | echo "all done." --------------------------------------------------------------------------------