├── .github ├── ISSUE_TEMPLATE │ ├── config.yaml │ ├── question.md │ ├── feature_request.md │ └── bug_report.md └── workflows │ ├── release.yml │ ├── go.yml │ ├── build-dev.yml │ ├── goreleaser.yml │ └── ci.yml ├── config ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── certmanager │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── certificate.yaml ├── webhook │ ├── kustomization.yaml │ ├── service.yaml │ ├── kustomizeconfig.yaml │ └── manifests.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── role_binding.yaml │ ├── auth_proxy_role_binding.yaml │ ├── leader_election_role_binding.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_service.yaml │ ├── kustomization.yaml │ ├── workload_viewer_role.yaml │ ├── workload_editor_role.yaml │ ├── leader_election_role.yaml │ └── role.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── samples │ ├── oam │ │ ├── workloaddefinitions.yaml │ │ ├── sample_application_config.yaml │ │ ├── sample_component.yaml │ │ └── oam.md │ └── cache_v1alpha1_workload.yaml ├── crd │ ├── patches │ │ ├── cainjection_in_workloads.yaml │ │ └── webhook_in_workloads.yaml │ ├── kustomizeconfig.yaml │ └── kustomization.yaml └── default │ ├── manager_webhook_patch.yaml │ ├── webhookcainjection_patch.yaml │ ├── manager_auth_proxy_patch.yaml │ └── kustomization.yaml ├── agent ├── g │ ├── const.go │ ├── g.go │ ├── logger.go │ └── cfg.go ├── config │ └── agent.json ├── go.mod ├── Dockerfile ├── main.go ├── api │ └── base.go ├── http │ ├── routes.go │ ├── http.go │ └── run.go └── go.sum ├── PROJECT ├── controllers ├── workload │ ├── common │ │ ├── observer │ │ │ ├── const.go │ │ │ ├── finalizer.go │ │ │ ├── watch.go │ │ │ ├── manager.go │ │ │ └── observer.go │ │ ├── cm │ │ │ ├── agent.go │ │ │ ├── cm.go │ │ │ ├── config.go │ │ │ ├── const.go │ │ │ ├── new.go │ │ │ └── init_script.go │ │ ├── zk │ │ │ ├── const.go │ │ │ ├── client.go │ │ │ └── base.go │ │ ├── sts │ │ │ ├── sts.go │ │ │ ├── volume.go │ │ │ ├── const.go │ │ │ ├── initcontainer.go │ │ │ ├── resource.go │ │ │ └── new.go │ │ ├── svc │ │ │ ├── svc.go │ │ │ └── new.go │ │ ├── prometheus │ │ │ ├── registry.go │ │ │ └── generic_client.go │ │ ├── volume │ │ │ ├── const.go │ │ │ ├── volume.go │ │ │ └── default.go │ │ ├── utils │ │ │ ├── stringutil.go │ │ │ └── pod.go │ │ ├── monitor │ │ │ └── monitor.go │ │ └── finalizer │ │ │ └── finalizer.go │ ├── provision │ │ ├── finalizer.go │ │ ├── monitor.go │ │ ├── sts.go │ │ ├── svc.go │ │ ├── provision.go │ │ ├── cm.go │ │ └── observer.go │ ├── scale │ │ ├── scale.go │ │ ├── sts.go │ │ ├── upscale.go │ │ ├── downscale.go │ │ └── reconfig.go │ ├── rollout │ │ ├── rollout.go │ │ └── rollingupdate.go │ ├── workload.go │ ├── model │ │ └── const.go │ └── getter.go ├── k8s │ ├── k8s_suite_test.go │ ├── discovery.go │ ├── new.go │ ├── client.go │ ├── client_test.go │ └── dynamic.go ├── merge_labels.go ├── getter.go ├── suite_test.go └── workload_controller.go ├── OWNERS.md ├── .gitignore ├── hack └── boilerplate.go.txt ├── Dockerfile ├── go.mod ├── api └── v1alpha1 │ ├── groupversion_info.go │ └── workload_webhook.go ├── docs └── DEVELOPMENT.md ├── Makefile ├── CODE_OF_CONDUCT.md ├── README.md ├── main.go └── manifests └── deploy.yaml /.github/ISSUE_TEMPLATE/config.yaml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /agent/g/const.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | const ( 4 | VERSION = "0.0.1" 5 | ) 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: ghostbaby.io 2 | repo: github.com/ghostbaby/zookeeper-operator 3 | resources: 4 | - group: zk.cache 5 | kind: Workload 6 | version: v1alpha1 7 | version: "2" 8 | -------------------------------------------------------------------------------- /agent/g/g.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "log" 5 | "runtime" 6 | ) 7 | 8 | func init() { 9 | runtime.GOMAXPROCS(runtime.NumCPU()) 10 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 11 | } 12 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: ["/metrics"] 7 | verbs: ["get"] 8 | -------------------------------------------------------------------------------- /controllers/workload/common/observer/const.go: -------------------------------------------------------------------------------- 1 | package observer 2 | 3 | import "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 4 | 5 | type State struct { 6 | ClusterStats *zk.ClusterStats 7 | } 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Help wanted. 4 | title: "[Question]" 5 | labels: help wanted 6 | assignees: '' 7 | 8 | --- 9 | 10 | 13 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | images: 6 | - name: controller 7 | newName: ghostbaby/zookeeper-operator 8 | newTag: dev 9 | -------------------------------------------------------------------------------- /agent/config/agent.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": true, 3 | "zkHost": "127.0.0.1", 4 | "zkPort": "2181", 5 | "http": { 6 | "enabled": true, 7 | "listen": ":1988", 8 | "backdoor": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /config/samples/oam/workloaddefinitions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: core.oam.dev/v1alpha2 2 | kind: WorkloadDefinition 3 | metadata: 4 | name: workloads.zk.cache.ghostbaby.io 5 | spec: 6 | definitionRef: 7 | name: workloads.zk.cache.ghostbaby.io -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 9443 11 | selector: 12 | control-plane: controller-manager 13 | -------------------------------------------------------------------------------- /controllers/k8s/k8s_suite_test.go: -------------------------------------------------------------------------------- 1 | package k8s_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestK8s(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "K8s Suite") 13 | } 14 | -------------------------------------------------------------------------------- /OWNERS.md: -------------------------------------------------------------------------------- 1 | # OWNERS 2 | 3 | This page lists all maintainers for **this** repository. Each repository in the [Ghostbaby](https://github.com/Ghostbaby/) will list their repository maintainers in their own 4 | `OWNERS.md` file. 5 | 6 | ## Maintainers 7 | 8 | * Huijun Zhu ([Ghostbaby](https://github.com/Ghostbaby)) -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: ["authentication.k8s.io"] 7 | resources: 8 | - tokenreviews 9 | verbs: ["create"] 10 | - apiGroups: ["authorization.k8s.io"] 11 | resources: 12 | - subjectaccessreviews 13 | verbs: ["create"] 14 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /agent/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ghostbaby/zk-agent 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/gorilla/mux v1.7.3 7 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da 8 | github.com/sirupsen/logrus v1.4.2 9 | github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07 10 | github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /controllers/workload/common/cm/agent.go: -------------------------------------------------------------------------------- 1 | package cm 2 | 3 | const ( 4 | AgentConfigKey = "config.json" 5 | ) 6 | 7 | func GenZkAgentConfig() string { 8 | return `{ 9 | "debug": true, 10 | "zkHost": "127.0.0.1", 11 | "zkPort": "2181", 12 | "http": { 13 | "enabled": true, 14 | "listen": ":1988", 15 | "backdoor": true 16 | } 17 | }` 18 | } 19 | -------------------------------------------------------------------------------- /controllers/workload/common/zk/const.go: -------------------------------------------------------------------------------- 1 | package zk 2 | 3 | import "time" 4 | 5 | const ( 6 | // DefaultVotingConfigExclusionsTimeout is the default timeout for setting voting exclusions. 7 | DefaultVotingConfigExclusionsTimeout = "30s" 8 | // DefaultReqTimeout is the default timeout used when performing HTTP calls against Elasticsearch 9 | DefaultReqTimeout = 3 * time.Minute 10 | ) 11 | -------------------------------------------------------------------------------- /agent/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine as builder 2 | ENV GO111MODULE=on 3 | ENV GOPROXY=https://goproxy.io 4 | RUN apk add --no-cache git 5 | WORKDIR /usr/src/zk-agent 6 | COPY . /usr/src/zk-agent 7 | RUN go build -v 8 | 9 | FROM alpine:3.10 10 | COPY --from=builder /usr/src/zk-agent/zk-agent /usr/local/bin/zk-agent 11 | ENTRYPOINT ["/usr/local/bin/zk-agent"] 12 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_workloads.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: workloads.zk.cache.ghostbaby.io 9 | -------------------------------------------------------------------------------- /controllers/workload/common/sts/sts.go: -------------------------------------------------------------------------------- 1 | package sts 2 | 3 | import cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 4 | 5 | type STS struct { 6 | Workload *cachev1alpha1.Workload 7 | Labels map[string]string 8 | } 9 | 10 | func NewSTS(workload *cachev1alpha1.Workload, labels map[string]string) *STS { 11 | return &STS{ 12 | Workload: workload, 13 | Labels: labels, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /controllers/workload/common/svc/svc.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 4 | 5 | type SVC struct { 6 | Workload *cachev1alpha1.Workload 7 | Labels map[string]string 8 | } 9 | 10 | func NewSVC(workload *cachev1alpha1.Workload, labels map[string]string) *SVC { 11 | return &SVC{ 12 | Workload: workload, 13 | Labels: labels, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /agent/g/logger.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import log "github.com/sirupsen/logrus" 4 | 5 | func InitLog(level string) (err error) { 6 | switch level { 7 | case "info": 8 | log.SetLevel(log.InfoLevel) 9 | case "debug": 10 | log.SetLevel(log.DebugLevel) 11 | case "warn": 12 | log.SetLevel(log.WarnLevel) 13 | default: 14 | log.Fatal("log conf only allow [info, debug, warn], please check your confguire") 15 | } 16 | return 17 | } 18 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | selector: 15 | matchLabels: 16 | control-plane: controller-manager 17 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - role.yaml 3 | - role_binding.yaml 4 | - leader_election_role.yaml 5 | - leader_election_role_binding.yaml 6 | # Comment the following 4 lines if you want to disable 7 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 8 | # which protects your /metrics endpoint. 9 | - auth_proxy_service.yaml 10 | - auth_proxy_role.yaml 11 | - auth_proxy_role_binding.yaml 12 | - auth_proxy_client_clusterrole.yaml 13 | -------------------------------------------------------------------------------- /controllers/workload/common/sts/volume.go: -------------------------------------------------------------------------------- 1 | package sts 2 | 3 | import ( 4 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/volume" 5 | corev1 "k8s.io/api/core/v1" 6 | ) 7 | 8 | func BuildDefaultVolumes() []corev1.VolumeMount { 9 | var volumeMounts []corev1.VolumeMount 10 | volumeMounts = append(volumeMounts, 11 | volume.DefaultDataVolumeMount, 12 | volume.DefaultLogsVolumeMount, 13 | ) 14 | 15 | return volumeMounts 16 | } 17 | -------------------------------------------------------------------------------- /controllers/workload/common/sts/const.go: -------------------------------------------------------------------------------- 1 | package sts 2 | 3 | const ( 4 | // prepareFilesystemContainerName is the name of the container that prepares the filesystem 5 | PrepareFilesystemContainerName = "zookeeper-internal-init-filesystem" 6 | ) 7 | 8 | const ( 9 | // EnvPodName and EnvPodIP are injected as env var into the ZK pod at runtime, 10 | // to be referenced in ZK configuration file 11 | EnvPodName = "POD_NAME" 12 | EnvPodIP = "POD_IP" 13 | ) 14 | -------------------------------------------------------------------------------- /config/rbac/workload_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view workloads. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: workload-viewer-role 6 | rules: 7 | - apiGroups: 8 | - zk.cache.ghostbaby.io 9 | resources: 10 | - workloads 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - zk.cache.ghostbaby.io 17 | resources: 18 | - workloads/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Kubernetes Generated files - skip generated files, except for vendored files 17 | 18 | !vendor/**/zz_generated.* 19 | 20 | # editor and IDE paraphernalia 21 | .idea 22 | *.swp 23 | *.swo 24 | *~ 25 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/rbac/workload_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit workloads. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: workload-editor-role 6 | rules: 7 | - apiGroups: 8 | - zk.cache.ghostbaby.io 9 | resources: 10 | - workloads 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - zk.cache.ghostbaby.io 21 | resources: 22 | - workloads/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /controllers/workload/common/prometheus/registry.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "sigs.k8s.io/controller-runtime/pkg/manager" 5 | ) 6 | 7 | var ( 8 | genericClient *GenericClientset 9 | ) 10 | 11 | func NewRegistry(mgr manager.Manager) (*GenericClientset, error) { 12 | var err error 13 | genericClient, err = newForConfig(mgr.GetConfig()) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return genericClient, nil 18 | } 19 | 20 | func GetGenericClient() GenericClientset { 21 | return *genericClient 22 | } 23 | -------------------------------------------------------------------------------- /config/samples/oam/sample_application_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: core.oam.dev/v1alpha2 2 | kind: ApplicationConfiguration 3 | metadata: 4 | name: zk-appconfig 5 | spec: 6 | components: 7 | - componentName: zk-component 8 | parameterValues: 9 | - name: name 10 | value: ghostbaby 11 | traits: 12 | - trait: 13 | apiVersion: core.oam.dev/v1alpha2 14 | kind: ManualScalerTrait 15 | metadata: 16 | name: zk-appconfig-trait 17 | spec: 18 | replicaCount: 3 -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | group: apiextensions.k8s.io 8 | path: spec/conversion/webhookClientConfig/service/name 9 | 10 | namespace: 11 | - kind: CustomResourceDefinition 12 | group: apiextensions.k8s.io 13 | path: spec/conversion/webhookClientConfig/service/namespace 14 | create: false 15 | 16 | varReference: 17 | - path: metadata/annotations 18 | -------------------------------------------------------------------------------- /controllers/workload/provision/finalizer.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 5 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 7 | ) 8 | 9 | func (p *Provision) FinalizersFor( 10 | zk *cachev1alpha1.Workload, 11 | ) []finalizer.Finalizer { 12 | clusterName := utils.ExtractNamespacedName(zk) 13 | return []finalizer.Finalizer{ 14 | p.Observers.Finalizer(clusterName), 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - configmaps/status 23 | verbs: 24 | - get 25 | - update 26 | - patch 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - events 31 | verbs: 32 | - create 33 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /agent/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/ghostbaby/zk-agent/g" 9 | "github.com/ghostbaby/zk-agent/http" 10 | ) 11 | 12 | func main() { 13 | 14 | cfg := flag.String("c", "cfg.json", "configuration file") 15 | version := flag.Bool("v", false, "show version") 16 | 17 | flag.Parse() 18 | 19 | if *version { 20 | fmt.Println(g.VERSION) 21 | os.Exit(0) 22 | } 23 | 24 | g.ParseConfig(*cfg) 25 | 26 | if g.Config().Debug { 27 | g.InitLog("debug") 28 | } else { 29 | g.InitLog("info") 30 | } 31 | 32 | go http.Start() 33 | 34 | select {} 35 | 36 | } 37 | -------------------------------------------------------------------------------- /controllers/workload/common/cm/cm.go: -------------------------------------------------------------------------------- 1 | package cm 2 | 3 | import ( 4 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 5 | appsv1 "k8s.io/api/apps/v1" 6 | ) 7 | 8 | type CM struct { 9 | Workload *cachev1alpha1.Workload 10 | Labels map[string]string 11 | ExpectSts *appsv1.StatefulSet 12 | ActualSts *appsv1.StatefulSet 13 | } 14 | 15 | func NewCM(workload *cachev1alpha1.Workload, labels map[string]string, expectSTS, actualSTS *appsv1.StatefulSet) *CM { 16 | return &CM{ 17 | Workload: workload, 18 | Labels: labels, 19 | ExpectSts: expectSTS, 20 | ActualSts: actualSTS, 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | ports: 12 | - containerPort: 9443 13 | name: webhook-server 14 | protocol: TCP 15 | volumeMounts: 16 | - mountPath: /tmp/k8s-webhook-server/serving-certs 17 | name: cert 18 | readOnly: true 19 | volumes: 20 | - name: cert 21 | secret: 22 | defaultMode: 420 23 | secretName: webhook-server-cert 24 | -------------------------------------------------------------------------------- /config/samples/oam/sample_component.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: core.oam.dev/v1alpha2 2 | kind: Component 3 | metadata: 4 | name: zk-component 5 | spec: 6 | workload: 7 | apiVersion: zk.cache.ghostbaby.io/v1alpha1 8 | kind: Workload 9 | spec: 10 | version: v3.5.6 11 | cluster: 12 | name: test 13 | resources: 14 | requests: 15 | cpu: 100m 16 | memory: 500Mi 17 | exporter: 18 | exporter: true 19 | exporterImage: ghostbaby/zookeeper_exporter 20 | exporterVersion: v3.5.6 21 | disableExporterProbes: false 22 | parameters: 23 | - name: name 24 | fieldPaths: 25 | - metadata.name -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | apiVersion: admissionregistration.k8s.io/v1beta1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | name: mutating-webhook-configuration 7 | annotations: 8 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 9 | --- 10 | apiVersion: admissionregistration.k8s.io/v1beta1 11 | kind: ValidatingWebhookConfiguration 12 | metadata: 13 | name: validating-webhook-configuration 14 | annotations: 15 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 16 | -------------------------------------------------------------------------------- /controllers/workload/common/cm/config.go: -------------------------------------------------------------------------------- 1 | package cm 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // PodName returns the name of the pod with the given ordinal for this StatefulSet. 8 | func PodName(ssetName string, ordinal int32) string { 9 | return fmt.Sprintf("%s-%d", ssetName, ordinal) 10 | } 11 | 12 | func GenZkConfig() string { 13 | return `tickTime=2000 14 | initLimit=10 15 | skipACL=yes 16 | syncLimit=5 17 | dataDir=/data 18 | maxClientCnxns=300 19 | dataLogDir=/logs 20 | reconfigEnabled=true 21 | standaloneEnabled=false 22 | autopurge.snapRetainCount=20 23 | autopurge.purgeInterval=24 24 | 4lw.commands.whitelist=cons, envi, conf, crst, srvr, stat, mntr, ruok 25 | dynamicConfigFile=/conf/zoo.cfg.dynamic 26 | ` 27 | } 28 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_workloads.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: workloads.zk.cache.ghostbaby.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - created 5 | - published 6 | 7 | jobs: 8 | docker: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | # opspresso/action-docker use body of ./target/TAG_NAME as docker tag 13 | # https://github.com/opspresso/action-docker#common-env 14 | - run: mkdir ./target 15 | - run: echo -n '${{ github.ref }}' | sed 's|refs/tags/||' > ./target/TAG_NAME 16 | - name: Docker Build & Push to Docker Hub 17 | uses: opspresso/action-docker@master 18 | with: 19 | args: --docker 20 | env: 21 | USERNAME: '${{ secrets.DOCKER_USER }}' 22 | PASSWORD: '${{ secrets.DOCKER_TOKEN }}' 23 | LATEST: 'false' -------------------------------------------------------------------------------- /controllers/workload/common/observer/finalizer.go: -------------------------------------------------------------------------------- 1 | package observer 2 | 3 | import ( 4 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 5 | "k8s.io/apimachinery/pkg/types" 6 | ) 7 | 8 | const ( 9 | // FinalizerName registered for each elasticsearch resource 10 | FinalizerName = "finalizer.zookeeper.ghostbaby.io/observer" 11 | ) 12 | 13 | // Finalizer returns a finalizer to be executed upon deletion of the given cluster, 14 | // that makes sure the cluster is not observed anymore 15 | func (m *Manager) Finalizer(cluster types.NamespacedName) finalizer.Finalizer { 16 | return finalizer.Finalizer{ 17 | Name: FinalizerName, 18 | Execute: func() error { 19 | m.StopObserving(cluster) 20 | return nil 21 | }, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /agent/api/base.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | ) 7 | 8 | func handleInternalServerError(w http.ResponseWriter) { 9 | http.Error(w, http.StatusText(http.StatusInternalServerError), 10 | http.StatusInternalServerError) 11 | } 12 | 13 | func handleUnauthorized(w http.ResponseWriter) { 14 | http.Error(w, http.StatusText(http.StatusUnauthorized), 15 | http.StatusUnauthorized) 16 | } 17 | 18 | // response status code will be written automatically if there is an error 19 | func WriteJSON(w http.ResponseWriter, v interface{}) error { 20 | b, err := json.Marshal(v) 21 | if err != nil { 22 | handleInternalServerError(w) 23 | return err 24 | } 25 | 26 | if _, err = w.Write(b); err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[Feature]" 5 | labels: kind/feature 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | 14 | 15 | **Describe the solution you'd like** 16 | 19 | 20 | **Describe alternatives you've considered** 21 | 24 | 25 | **Additional context** 26 | 29 | -------------------------------------------------------------------------------- /controllers/workload/common/volume/const.go: -------------------------------------------------------------------------------- 1 | package volume 2 | 3 | const ( 4 | DataVolClaimName = "zookeeper-data" 5 | AffinityOff = "none" 6 | ) 7 | 8 | const ( 9 | ConfigFileName = "zoo.cfg" 10 | ConfigVolumeName = "zookeeper-internal-config" 11 | ConfigVolumeMountPath = "/conf" 12 | 13 | DynamicConfigFileVolumeName = "zookeeper-internal-dynamic-config" 14 | DynamicConfigFileVolumeMountPath = "/mnt/zookeeper/dynamic-config" 15 | DynamicConfigFile = "zoo_replicated1.cfg.dynamic" 16 | 17 | DataVolumeName = "zookeeper-data" 18 | DataMountPath = "/data" 19 | 20 | LogsVolumeName = "zookeeper-logs" 21 | LogsMountPath = "/logs" 22 | 23 | ScriptsVolumeName = "zookeeper-internal-scripts" 24 | ScriptsVolumeMountPath = "/mnt/zookeeper/scripts" 25 | ) 26 | -------------------------------------------------------------------------------- /controllers/merge_labels.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 5 | ) 6 | 7 | func GenerateLabels(labels map[string]string, name string) map[string]string { 8 | dynLabels := map[string]string{ 9 | model.AppLabel: name, 10 | "app.kubernetes.io/name": "zookeeper", 11 | "app.kubernetes.io/instance": name, 12 | "app.kubernetes.io/managed-by": "zookeeper-operator", 13 | "app.kubernetes.io/part-of": "zookeeper", 14 | } 15 | return MergeLabels(dynLabels, labels) 16 | } 17 | 18 | func MergeLabels(allLabels ...map[string]string) map[string]string { 19 | res := map[string]string{} 20 | 21 | for _, labels := range allLabels { 22 | for k, v := range labels { 23 | res[k] = v 24 | } 25 | } 26 | return res 27 | } 28 | -------------------------------------------------------------------------------- /controllers/workload/common/cm/const.go: -------------------------------------------------------------------------------- 1 | package cm 2 | 3 | const ( 4 | ConfigFileName = "zoo.cfg" 5 | ConfigVolumeName = "zookeeper-internal-config" 6 | ConfigVolumeMountPath = "/mnt/zookeeper/zookeeper-config" 7 | 8 | DynamicConfigFileVolumeName = "zookeeper-internal-dynamic-config" 9 | DynamicConfigFileVolumeMountPath = "/mnt/zookeeper/dynamic-config" 10 | DynamicConfigFile = "zoo_replicated1.cfg.dynamic" 11 | 12 | DataVolumeName = "zookeeper-data" 13 | DataMountPath = "/data" 14 | 15 | LogsVolumeName = "zookeeper-logs" 16 | LogsMountPath = "/logs" 17 | 18 | ScriptsVolumeName = "zookeeper-internal-scripts" 19 | ScriptsVolumeMountPath = "/mnt/zookeeper/scripts" 20 | 21 | AgentVolumeName = "zookeeper-agent-config" 22 | AgentVolumeMountPath = "/mnt/zookeeper/agent" 23 | ) 24 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: Build 34 | run: go build -v . 35 | 36 | - name: Test 37 | run: go test -v . 38 | -------------------------------------------------------------------------------- /controllers/workload/common/prometheus/generic_client.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | poclientset "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" 5 | kubeclientset "k8s.io/client-go/kubernetes" 6 | "k8s.io/client-go/rest" 7 | ) 8 | 9 | type GenericClientset struct { 10 | KubeClient kubeclientset.Interface 11 | PoClient poclientset.Interface 12 | } 13 | 14 | // NewForConfig creates a new Clientset for the given config. 15 | func newForConfig(c *rest.Config) (*GenericClientset, error) { 16 | kubeClient, err := kubeclientset.NewForConfig(c) 17 | if err != nil { 18 | return nil, err 19 | } 20 | kruiseClient, err := poclientset.NewForConfig(c) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return &GenericClientset{ 25 | KubeClient: kubeClient, 26 | PoClient: kruiseClient, 27 | }, nil 28 | } 29 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | name: https 22 | - name: manager 23 | args: 24 | - "--metrics-addr=127.0.0.1:8080" 25 | - "--enable-leader-election" 26 | -------------------------------------------------------------------------------- /config/samples/cache_v1alpha1_workload.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: zk.cache.ghostbaby.io/v1alpha1 2 | kind: Workload 3 | metadata: 4 | name: workload-sample 5 | # namespace: pg 6 | spec: 7 | version: v3.5.6 8 | replicas: 3 9 | cluster: 10 | name: test 11 | resources: 12 | requests: 13 | cpu: 500m 14 | memory: 1Gi 15 | limits: 16 | cpu: 500m 17 | memory: 2Gi 18 | exporter: 19 | exporter: true 20 | exporterImage: ghostbaby/zookeeper_exporter 21 | exporterVersion: v3.5.6 22 | disableExporterProbes: false 23 | storage: 24 | persistentVolumeClaim: 25 | metadata: 26 | name: zookeeper-data 27 | spec: 28 | accessModes: 29 | - ReadWriteOnce 30 | resources: 31 | requests: 32 | storage: 1Gi 33 | # storageClassName: nfs-storage 34 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: kind/bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 14 | 15 | **To Reproduce** 16 | 23 | 24 | **Expected behavior** 25 | 28 | 29 | **Screenshots** 30 | 33 | 34 | **Cluster information** 35 | 39 | 40 | **Additional context** 41 | 44 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.13 as builder 3 | 4 | WORKDIR /workspace 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | ENV GOPROXY=https://goproxy.io 9 | # cache deps before building and copying source so that we don't need to re-download as much 10 | # and so that source changes don't invalidate our downloaded layer 11 | RUN go mod download 12 | 13 | # Copy the go source 14 | COPY main.go main.go 15 | COPY api/ api/ 16 | COPY controllers/ controllers/ 17 | 18 | # Build 19 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go 20 | 21 | # Use distroless as minimal base image to package the manager binary 22 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 23 | FROM gcr.io/distroless/static:nonroot 24 | WORKDIR / 25 | COPY --from=builder /workspace/manager . 26 | USER nonroot:nonroot 27 | 28 | ENTRYPOINT ["/manager"] 29 | -------------------------------------------------------------------------------- /controllers/workload/common/volume/volume.go: -------------------------------------------------------------------------------- 1 | package volume 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | ) 6 | 7 | type ConfigMapVolume struct { 8 | CmName string 9 | Name string 10 | MountPath string 11 | DefaultMode int32 12 | } 13 | 14 | var ( 15 | defaultOptional = false 16 | ) 17 | 18 | func (cm ConfigMapVolume) Volume() corev1.Volume { 19 | return corev1.Volume{ 20 | Name: cm.Name, 21 | VolumeSource: corev1.VolumeSource{ 22 | ConfigMap: &corev1.ConfigMapVolumeSource{ 23 | LocalObjectReference: corev1.LocalObjectReference{ 24 | Name: cm.CmName, 25 | }, 26 | Optional: &defaultOptional, 27 | DefaultMode: &cm.DefaultMode, 28 | }, 29 | }, 30 | } 31 | } 32 | 33 | // VolumeMount returns the k8s volume mount. 34 | func (cm ConfigMapVolume) VolumeMount() corev1.VolumeMount { 35 | return corev1.VolumeMount{ 36 | Name: cm.Name, 37 | MountPath: cm.MountPath, 38 | ReadOnly: true, 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | labels: 23 | control-plane: controller-manager 24 | spec: 25 | containers: 26 | - command: 27 | - /manager 28 | args: 29 | - --enable-leader-election 30 | image: controller:latest 31 | name: manager 32 | resources: 33 | limits: 34 | cpu: 100m 35 | memory: 30Mi 36 | requests: 37 | cpu: 100m 38 | memory: 20Mi 39 | terminationGracePeriodSeconds: 10 40 | -------------------------------------------------------------------------------- /controllers/workload/scale/scale.go: -------------------------------------------------------------------------------- 1 | package scale 2 | 3 | import ( 4 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 5 | "github.com/ghostbaby/zookeeper-operator/controllers/k8s" 6 | "github.com/go-logr/logr" 7 | appsv1 "k8s.io/api/apps/v1" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | "k8s.io/client-go/tools/record" 10 | ) 11 | 12 | type Scale struct { 13 | Workload *cachev1alpha1.Workload 14 | Client k8s.Client 15 | Recorder record.EventRecorder 16 | Log logr.Logger 17 | Labels map[string]string 18 | Scheme *runtime.Scheme 19 | ExpectSts *appsv1.StatefulSet 20 | ActualSts *appsv1.StatefulSet 21 | } 22 | 23 | func (s *Scale) Reconcile() error { 24 | 25 | if err := s.StatefulSet(); err != nil { 26 | return err 27 | } 28 | 29 | if err := s.UpScale(); err != nil { 30 | return err 31 | } 32 | 33 | if err := s.ReConfig(); err != nil { 34 | return err 35 | } 36 | 37 | if err := s.DownScale(); err != nil { 38 | return err 39 | } 40 | 41 | return nil 42 | } 43 | -------------------------------------------------------------------------------- /.github/workflows/build-dev.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - '**' 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | # Dockerfile tests 13 | docker-build-test: 14 | runs-on: ubuntu-latest 15 | name: Build and Test 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: install kubebuilder 20 | run: | 21 | curl -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_linux_amd64.tar.gz | tar -xz -C /tmp/ 22 | sudo mv /tmp/kubebuilder_2.3.1_linux_amd64 /usr/local/kubebuilder 23 | - run: make test docker-build 24 | - name: Docker Build & Push to Docker Hub 25 | uses: opspresso/action-docker@master 26 | with: 27 | args: --docker 28 | env: 29 | USERNAME: ${{ secrets.DOCKER_USER }} 30 | PASSWORD: ${{ secrets.DOCKER_TOKEN }} 31 | DOCKERFILE: Dockerfile 32 | IMAGE_NAME: ghostbaby/zookeeper-operator 33 | TAG_NAME: dev 34 | LATEST: 'false' 35 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for 4 | # breaking changes 5 | apiVersion: cert-manager.io/v1alpha2 6 | kind: Issuer 7 | metadata: 8 | name: selfsigned-issuer 9 | namespace: system 10 | spec: 11 | selfSigned: {} 12 | --- 13 | apiVersion: cert-manager.io/v1alpha2 14 | kind: Certificate 15 | metadata: 16 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 17 | namespace: system 18 | spec: 19 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 20 | dnsNames: 21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 22 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 23 | issuerRef: 24 | kind: Issuer 25 | name: selfsigned-issuer 26 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 27 | -------------------------------------------------------------------------------- /controllers/workload/scale/sts.go: -------------------------------------------------------------------------------- 1 | package scale 2 | 3 | import ( 4 | appsv1 "k8s.io/api/apps/v1" 5 | 6 | "k8s.io/apimachinery/pkg/api/errors" 7 | "k8s.io/apimachinery/pkg/types" 8 | 9 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/sts" 10 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 11 | ) 12 | 13 | func (s *Scale) StatefulSet() error { 14 | actual := &appsv1.StatefulSet{} 15 | name := s.Workload.Name 16 | namespace := s.Workload.Namespace 17 | 18 | newSts := sts.NewSTS(s.Workload, s.Labels) 19 | 20 | expect, err := newSts.GenerateStatefulset() 21 | if err != nil { 22 | return err 23 | } 24 | if err := controllerutil.SetControllerReference(s.Workload, expect, s.Scheme); err != nil { 25 | return err 26 | } 27 | 28 | if err := s.Client.Get(types.NamespacedName{Name: name, Namespace: namespace}, actual); err != nil && errors.IsNotFound(err) { 29 | s.ExpectSts = expect 30 | } else if err != nil { 31 | return err 32 | } else { 33 | s.ExpectSts = expect 34 | s.ActualSts = actual 35 | } 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - 'v*' 5 | 6 | jobs: 7 | goreleaser: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - 11 | name: Checkout 12 | uses: actions/checkout@v2 13 | - 14 | name: Unshallow 15 | # required for the changelog to work correctly 16 | run: git fetch --prune --unshallow 17 | - 18 | name: Set up Go 19 | uses: actions/setup-go@v1 20 | with: 21 | go-version: 1.13 22 | - 23 | name: prepare changelog 24 | run: | 25 | tag=${{ github.ref }} 26 | tag=${tag##*/} 27 | cat < k8s.io/client-go v0.0.0-20200813012017-e7a1d9ada0d5 28 | 29 | replace github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring => github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.42.1 30 | -------------------------------------------------------------------------------- /agent/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | _ "net/http/pprof" 8 | 9 | "github.com/ghostbaby/zk-agent/g" 10 | ) 11 | 12 | type Dto struct { 13 | Msg string `json:"msg"` 14 | Data interface{} `json:"data"` 15 | } 16 | 17 | func RenderJson(w http.ResponseWriter, v interface{}) { 18 | bs, err := json.Marshal(v) 19 | if err != nil { 20 | http.Error(w, err.Error(), http.StatusInternalServerError) 21 | return 22 | } 23 | 24 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 25 | w.Write(bs) 26 | } 27 | 28 | func RenderDataJson(w http.ResponseWriter, data interface{}) { 29 | RenderJson(w, Dto{Msg: "success", Data: data}) 30 | } 31 | 32 | func RenderMsgJson(w http.ResponseWriter, msg string) { 33 | RenderJson(w, map[string]string{"msg": msg}) 34 | } 35 | 36 | func AutoRender(w http.ResponseWriter, data interface{}, err error) { 37 | if err != nil { 38 | RenderMsgJson(w, err.Error()) 39 | return 40 | } 41 | 42 | RenderDataJson(w, data) 43 | } 44 | 45 | func Start() { 46 | if !g.Config().Http.Enabled { 47 | return 48 | } 49 | 50 | addr := g.Config().Http.Listen 51 | if addr == "" { 52 | return 53 | } 54 | 55 | r := NewRouter() 56 | 57 | log.Println("listening", addr) 58 | log.Fatalln(http.ListenAndServe(addr, r)) 59 | } 60 | -------------------------------------------------------------------------------- /config/webhook/manifests.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: admissionregistration.k8s.io/v1beta1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | creationTimestamp: null 7 | name: mutating-webhook-configuration 8 | webhooks: 9 | - clientConfig: 10 | caBundle: Cg== 11 | service: 12 | name: webhook-service 13 | namespace: system 14 | path: /mutate-zk-cache-ghostbaby-io-v1alpha1-workload 15 | failurePolicy: Fail 16 | name: mworkload.kb.io 17 | rules: 18 | - apiGroups: 19 | - zk.cache.ghostbaby.io 20 | apiVersions: 21 | - v1alpha1 22 | operations: 23 | - CREATE 24 | - UPDATE 25 | resources: 26 | - workloads 27 | 28 | --- 29 | apiVersion: admissionregistration.k8s.io/v1beta1 30 | kind: ValidatingWebhookConfiguration 31 | metadata: 32 | creationTimestamp: null 33 | name: validating-webhook-configuration 34 | webhooks: 35 | - clientConfig: 36 | caBundle: Cg== 37 | service: 38 | name: webhook-service 39 | namespace: system 40 | path: /validate-zk-cache-ghostbaby-io-v1alpha1-workload 41 | failurePolicy: Fail 42 | name: vworkload.kb.io 43 | rules: 44 | - apiGroups: 45 | - zk.cache.ghostbaby.io 46 | apiVersions: 47 | - v1alpha1 48 | operations: 49 | - CREATE 50 | - UPDATE 51 | resources: 52 | - workloads 53 | -------------------------------------------------------------------------------- /controllers/workload/common/utils/stringutil.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "strings" 4 | 5 | func Joins(args ...string) string { 6 | var str strings.Builder 7 | for _, arg := range args { 8 | str.WriteString(arg) 9 | } 10 | return str.String() 11 | } 12 | 13 | // StringsInSlice returns true if the given strings are found in the provided slice, else returns false 14 | func StringsInSlice(strings []string, slice []string) bool { 15 | asMap := make(map[string]struct{}, len(slice)) 16 | for _, s := range slice { 17 | asMap[s] = struct{}{} 18 | } 19 | for _, s := range strings { 20 | if _, exists := asMap[s]; !exists { 21 | return false 22 | } 23 | } 24 | return true 25 | } 26 | 27 | // StringInSlice returns true if the given string is found in the provided slice, else returns false 28 | func StringInSlice(str string, list []string) bool { 29 | for _, s := range list { 30 | if s == str { 31 | return true 32 | } 33 | } 34 | return false 35 | } 36 | 37 | // RemoveStringInSlice returns a new slice with all occurrences of s removed, 38 | // keeping the given slice unmodified 39 | func RemoveStringInSlice(s string, slice []string) []string { 40 | result := make([]string, 0, len(slice)) 41 | for _, item := range slice { 42 | if item == s { 43 | continue 44 | } 45 | result = append(result, item) 46 | } 47 | return result 48 | } 49 | -------------------------------------------------------------------------------- /controllers/workload/common/monitor/monitor.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | import ( 4 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 5 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 6 | monitorV1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | ) 9 | 10 | type Monitor struct { 11 | Workload *cachev1alpha1.Workload 12 | Labels map[string]string 13 | } 14 | 15 | func NewMonitor(workload *cachev1alpha1.Workload, labels map[string]string) *Monitor { 16 | return &Monitor{ 17 | Workload: workload, 18 | Labels: labels, 19 | } 20 | } 21 | 22 | func (m *Monitor) GenerateMongodbServiceMonitor() (*monitorV1.ServiceMonitor, error) { 23 | name := m.Workload.Name 24 | namespace := m.Workload.Namespace 25 | 26 | return &monitorV1.ServiceMonitor{ 27 | ObjectMeta: metav1.ObjectMeta{ 28 | Name: name, 29 | Namespace: namespace, 30 | Labels: m.Labels, 31 | }, 32 | Spec: monitorV1.ServiceMonitorSpec{ 33 | Endpoints: []monitorV1.Endpoint{ 34 | { 35 | Interval: model.ServiceMonitorInterval, 36 | Port: model.ServiceMonitorPort, 37 | }, 38 | }, 39 | Selector: metav1.LabelSelector{ 40 | MatchLabels: map[string]string{ 41 | model.RoleName: name, 42 | }, 43 | }, 44 | }, 45 | }, nil 46 | } 47 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the cache v1alpha1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=zk.cache.ghostbaby.io 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "zk.cache.ghostbaby.io", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /controllers/workload/scale/upscale.go: -------------------------------------------------------------------------------- 1 | package scale 2 | 3 | import ( 4 | "fmt" 5 | 6 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 9 | 10 | //appsv1 "k8s.io/api/apps/v1" 11 | corev1 "k8s.io/api/core/v1" 12 | ) 13 | 14 | func (s *Scale) UpScale() error { 15 | //[BUG] upscale.go:13 +0x51 集群刚刚启动，无法获取到期待sts，导致operator crush 16 | 17 | if s.ExpectSts == nil || s.ActualSts == nil { 18 | return nil 19 | } 20 | name := s.Workload.GetName() 21 | expectReplica := s.ExpectSts.Spec.Replicas 22 | actualReplica := s.ActualSts.Spec.Replicas 23 | 24 | if expectReplica != nil && actualReplica != nil && *expectReplica > *actualReplica { 25 | msg := fmt.Sprintf(model.UpdateMessageZooKeeperStatefulset, name) 26 | s.Recorder.Event(s.Workload, corev1.EventTypeNormal, model.ZooKeeperStatefulset, msg) 27 | 28 | s.Log.Info( 29 | "Scaling replicas up", 30 | "from", actualReplica, 31 | "to", expectReplica, 32 | ) 33 | 34 | s.Workload.Status.Phase = cachev1alpha1.ZooKeeperUpScaling 35 | msg = fmt.Sprintf(model.MessageZooKeeperUpScaling, actualReplica, expectReplica) 36 | s.Recorder.Event(s.Workload, corev1.EventTypeNormal, model.ZooKeeperUpScaling, msg) 37 | 38 | err := s.Client.Update(s.ExpectSts) 39 | if err != nil { 40 | return err 41 | } 42 | } 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /controllers/workload/scale/downscale.go: -------------------------------------------------------------------------------- 1 | package scale 2 | 3 | import ( 4 | "fmt" 5 | 6 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 9 | 10 | //appsv1 "k8s.io/api/apps/v1" 11 | corev1 "k8s.io/api/core/v1" 12 | ) 13 | 14 | func (s *Scale) DownScale() error { 15 | //[BUG] upscale.go:13 +0x51 集群刚刚启动，无法获取到期待sts，导致operator crush 16 | if s.ExpectSts == nil || s.ActualSts == nil { 17 | return nil 18 | } 19 | name := s.Workload.GetName() 20 | expectReplica := s.ExpectSts.Spec.Replicas 21 | actualReplica := s.ActualSts.Spec.Replicas 22 | 23 | if expectReplica != nil && actualReplica != nil && *expectReplica < *actualReplica { 24 | msg := fmt.Sprintf(model.UpdateMessageZooKeeperStatefulset, name) 25 | s.Recorder.Event(s.Workload, corev1.EventTypeNormal, model.ZooKeeperStatefulset, msg) 26 | 27 | s.Log.Info( 28 | "Scaling replicas down", 29 | "from", actualReplica, 30 | "to", expectReplica, 31 | ) 32 | 33 | s.Workload.Status.Phase = cachev1alpha1.ZooKeeperDownScaling 34 | msg = fmt.Sprintf(model.MessageZooKeeperDownScaling, actualReplica, expectReplica) 35 | s.Recorder.Event(s.Workload, corev1.EventTypeNormal, model.ZooKeeperDownScaling, msg) 36 | 37 | err := s.Client.Update(s.ExpectSts) 38 | if err != nil { 39 | return err 40 | } 41 | } 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /controllers/k8s/discovery.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | 6 | openapi "github.com/googleapis/gnostic/openapiv2" 7 | "k8s.io/client-go/discovery" 8 | ) 9 | 10 | // WrapClient returns a Client that performs requests within DefaultTimeout. 11 | func WrapDiscoveryClient(ctx context.Context, client discovery.DiscoveryClient) DisClient { 12 | return &ClusterDiscoveryClient{ 13 | crClient: client, 14 | ctx: ctx, 15 | } 16 | } 17 | 18 | // Client wraps a discovery client to use a 19 | // default context with a timeout if no context is passed. 20 | type DisClient interface { 21 | // WithContext returns a client configured to use the provided context on 22 | // subsequent requests, instead of one created from the preconfigured timeout. 23 | WithContext(ctx context.Context) DisClient 24 | 25 | OpenAPISchema() (*openapi.Document, error) 26 | } 27 | 28 | type ClusterDiscoveryClient struct { 29 | crClient discovery.DiscoveryClient 30 | ctx context.Context 31 | } 32 | 33 | // WithContext returns a client configured to use the provided context on 34 | // subsequent requests, instead of one created from the preconfigured timeout. 35 | func (w *ClusterDiscoveryClient) WithContext(ctx context.Context) DisClient { 36 | w.ctx = ctx 37 | return w 38 | } 39 | 40 | func (w *ClusterDiscoveryClient) OpenAPISchema() (*openapi.Document, error) { 41 | return w.crClient.OpenAPISchema() 42 | } 43 | -------------------------------------------------------------------------------- /controllers/workload/provision/monitor.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/monitor" 7 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 8 | corev1 "k8s.io/api/core/v1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 11 | ) 12 | 13 | func (p *Provision) ProvisionMonitor() error { 14 | 15 | name := p.Workload.Name 16 | namespace := p.Workload.Namespace 17 | 18 | m := monitor.NewMonitor(p.Workload, p.Labels) 19 | sm, err := m.GenerateMongodbServiceMonitor() 20 | if err != nil { 21 | return err 22 | } 23 | 24 | if err := controllerutil.SetControllerReference(p.Workload, sm, p.Scheme); err != nil { 25 | return err 26 | } 27 | 28 | if _, err := p.Monitor.PoClient.MonitoringV1().ServiceMonitors(namespace).Get(p.CTX, name, metav1.GetOptions{}); err != nil { 29 | p.Log.Info("Creating ServiceMonitor %s/%s\n", p.Workload.Namespace, p.Workload.Name) 30 | 31 | _, err := p.Monitor.PoClient.MonitoringV1().ServiceMonitors(namespace).Create(p.CTX, sm, metav1.CreateOptions{}) 32 | if err != nil { 33 | return err 34 | } 35 | msg := fmt.Sprintf(model.MessageZooKeeperServiceMonitor, name) 36 | p.Recorder.Event(p.Workload, corev1.EventTypeNormal, model.ZooKeeperServiceMonitor, msg) 37 | } 38 | 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /docs/DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | This doc explains how to set up a development environment, so you can get started 4 | contributing to `zookeeper-operator` or build a PoC (Proof of Concept). 5 | 6 | ## Prerequisites 7 | 8 | 1. Golang version 1.13+ 9 | 2. Kubernetes version v1.15+ with `~/.kube/config` configured. 10 | 4. Kustomize version 3.8+ 11 | 5. Kubebuilder version 2.0+ 12 | 13 | ## Build 14 | * Clone this project 15 | 16 | ```shell script 17 | git clone git@github.com:Ghostbaby/zookeeper-operator.git 18 | ``` 19 | 20 | * Install Zookeeper CRD into your cluster 21 | 22 | ```shell script 23 | make install 24 | ``` 25 | 26 | ## Develop & Debug 27 | If you change Zookeeper CRD, remember to rerun `make install`. 28 | 29 | Use the following command to develop and debug. 30 | 31 | ```shell script 32 | $ make run 33 | ``` 34 | 35 | For example, use the following command to create an zookeeper cluster. 36 | ```shell script 37 | $ cd ./config/samples 38 | 39 | $ kubectl apply -f cache_v1alpha1_workload.yaml 40 | workload.zk.cache.ghostbaby.io/workload-sample created 41 | 42 | $ kubectl get pods -n pg 43 | workload-sample-0 3/3 Running 0 3m 44 | workload-sample-1 3/3 Running 0 3m 45 | workload-sample-2 3/3 Running 0 3m 46 | ``` 47 | 48 | ## Make a pull request 49 | Remember to write unit-test and e2e test before making a pull request. 50 | -------------------------------------------------------------------------------- /controllers/workload/common/svc/new.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 5 | corev1 "k8s.io/api/core/v1" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | ) 8 | 9 | func (s *SVC) GenerateService(name string, svcType string) *corev1.Service { 10 | namespace := s.Workload.GetNamespace() 11 | var clusterIP string 12 | 13 | if svcType == "Headless" { 14 | clusterIP = "None" 15 | } 16 | 17 | return &corev1.Service{ 18 | ObjectMeta: metav1.ObjectMeta{ 19 | Name: name, 20 | Namespace: namespace, 21 | Labels: s.Labels, 22 | Annotations: map[string]string{ 23 | "prometheus.io/scrape": "true", 24 | "prometheus.io/port": "http", 25 | "prometheus.io/path": "/metrics", 26 | }, 27 | }, 28 | Spec: corev1.ServiceSpec{ 29 | Ports: []corev1.ServicePort{ 30 | { 31 | Port: model.ClientPort, 32 | Protocol: corev1.ProtocolTCP, 33 | Name: "client", 34 | }, 35 | { 36 | Name: "server", 37 | Port: model.ServerPort, 38 | Protocol: corev1.ProtocolTCP, 39 | }, 40 | { 41 | Name: "leader-election", 42 | Port: model.LeaderElectionPort, 43 | Protocol: corev1.ProtocolTCP, 44 | }, 45 | { 46 | Port: model.ExporterPort, 47 | Protocol: corev1.ProtocolTCP, 48 | Name: model.ExporterPortName, 49 | }, 50 | { 51 | Port: model.AgentPort, 52 | Protocol: corev1.ProtocolTCP, 53 | Name: model.AgentPortName, 54 | }, 55 | }, 56 | ClusterIP: clusterIP, 57 | Selector: s.Labels, 58 | }, 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /controllers/workload/common/volume/default.go: -------------------------------------------------------------------------------- 1 | package volume 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | "k8s.io/apimachinery/pkg/api/resource" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | ) 8 | 9 | var ( 10 | // DefaultDataVolumeClaim is the default data volume claim for Elasticsearch pods. 11 | // We default to a 1GB persistent volume, using the default storage class. 12 | DefaultDataVolumeClaim = corev1.PersistentVolumeClaim{ 13 | ObjectMeta: metav1.ObjectMeta{ 14 | Name: DataVolumeName, 15 | }, 16 | Spec: corev1.PersistentVolumeClaimSpec{ 17 | AccessModes: []corev1.PersistentVolumeAccessMode{ 18 | corev1.ReadWriteOnce, 19 | }, 20 | Resources: corev1.ResourceRequirements{ 21 | Requests: corev1.ResourceList{ 22 | corev1.ResourceStorage: resource.MustParse("1Gi"), 23 | }, 24 | }, 25 | }, 26 | } 27 | DefaultDataVolumeMount = corev1.VolumeMount{ 28 | Name: DataVolumeName, 29 | MountPath: DataMountPath, 30 | } 31 | 32 | // DefaultVolumeClaimTemplates is the default volume claim templates for Elasticsearch pods 33 | DefaultVolumeClaimTemplates = []corev1.PersistentVolumeClaim{DefaultDataVolumeClaim} 34 | 35 | // DefaultLogsVolume is the default EmptyDir logs volume for Elasticsearch pods. 36 | DefaultLogsVolume = corev1.Volume{ 37 | Name: LogsVolumeName, 38 | VolumeSource: corev1.VolumeSource{ 39 | EmptyDir: &corev1.EmptyDirVolumeSource{}, 40 | }, 41 | } 42 | // DefaultLogsVolumeMount is the default logs volume mount for the Elasticsearch container. 43 | DefaultLogsVolumeMount = corev1.VolumeMount{ 44 | Name: LogsVolumeName, 45 | MountPath: LogsMountPath, 46 | } 47 | ) 48 | -------------------------------------------------------------------------------- /controllers/workload/provision/sts.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | "fmt" 5 | 6 | appsv1 "k8s.io/api/apps/v1" 7 | 8 | "k8s.io/apimachinery/pkg/api/errors" 9 | "k8s.io/apimachinery/pkg/types" 10 | 11 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/sts" 12 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 13 | corev1 "k8s.io/api/core/v1" 14 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 15 | ) 16 | 17 | func (p *Provision) ProvisionStatefulset() error { 18 | actual := &appsv1.StatefulSet{} 19 | name := p.Workload.Name 20 | namespace := p.Workload.Namespace 21 | 22 | s := sts.NewSTS(p.Workload, p.Labels) 23 | 24 | expect, err := s.GenerateStatefulset() 25 | if err != nil { 26 | return err 27 | } 28 | if err := controllerutil.SetControllerReference(p.Workload, expect, p.Scheme); err != nil { 29 | return err 30 | } 31 | 32 | if err := p.Client.Get(types.NamespacedName{Name: name, Namespace: namespace}, actual); err != nil && errors.IsNotFound(err) { 33 | p.Log.Info("Creating StatefulSet.", 34 | "namespace", namespace, "name", name) 35 | 36 | if err := p.Client.Create(expect); err != nil { 37 | return err 38 | } 39 | 40 | p.ExpectSts = expect 41 | 42 | msg := fmt.Sprintf(model.MessageZooKeeperStatefulset, name) 43 | p.Recorder.Event(p.Workload, corev1.EventTypeNormal, model.ZooKeeperStatefulset, msg) 44 | 45 | p.Log.Info("StatefulSet create complete.", 46 | "namespace", namespace, "name", name) 47 | 48 | } else if err != nil { 49 | return err 50 | } else { 51 | p.ExpectSts = expect 52 | p.ActualSts = actual 53 | } 54 | 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /controllers/workload/provision/svc.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 7 | 8 | svc2 "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/svc" 9 | 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/api/errors" 12 | "k8s.io/apimachinery/pkg/types" 13 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 14 | ) 15 | 16 | // EnsureService makes sure the mongodb statefulset exists 17 | func (p *Provision) ProvisionService() error { 18 | service := &corev1.Service{} 19 | var svcs []*corev1.Service 20 | name := p.Workload.GetName() 21 | namespace := p.Workload.GetNamespace() 22 | 23 | s := svc2.NewSVC(p.Workload, p.Labels) 24 | 25 | svc := s.GenerateService(name, "ClusterIP") 26 | svcs = append(svcs, svc) 27 | 28 | svcHeadless := s.GenerateService(name+"-s", "Headless") 29 | svcs = append(svcs, svcHeadless) 30 | 31 | for _, dep := range svcs { 32 | if err := controllerutil.SetControllerReference(p.Workload, dep, p.Scheme); err != nil { 33 | p.Log.Error(err, "SVC set ownerReference fail.", "namespace", dep.Namespace, "name", dep.Name) 34 | return err 35 | } 36 | err := p.Client.Get(types.NamespacedName{Name: dep.Name, Namespace: namespace}, service) 37 | if err != nil && errors.IsNotFound(err) { 38 | p.Log.Info("Creating Service.", "namespace", namespace, "name", dep.Name) 39 | err = p.Client.Create(dep) 40 | if err != nil { 41 | return err 42 | } 43 | msg := fmt.Sprintf(model.MessageZooKeeperService, name) 44 | p.Recorder.Event(p.Workload, corev1.EventTypeNormal, model.ZooKeeperService, msg) 45 | p.Log.Info("Service create complete.", "namespace", namespace, "name", dep.Name) 46 | } 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - configmaps 13 | - events 14 | - pods 15 | - secret 16 | - services 17 | verbs: 18 | - create 19 | - delete 20 | - get 21 | - list 22 | - patch 23 | - update 24 | - watch 25 | - apiGroups: 26 | - "" 27 | resources: 28 | - pods/exec 29 | verbs: 30 | - create 31 | - apiGroups: 32 | - apiextensions.k8s.io 33 | resources: 34 | - customresourcedefinitions 35 | verbs: 36 | - get 37 | - list 38 | - apiGroups: 39 | - apps 40 | resources: 41 | - statefulsets 42 | verbs: 43 | - create 44 | - delete 45 | - get 46 | - list 47 | - patch 48 | - update 49 | - watch 50 | - apiGroups: 51 | - apps 52 | resources: 53 | - statefulsets/status 54 | verbs: 55 | - get 56 | - patch 57 | - update 58 | - apiGroups: 59 | - monitoring.coreos.com 60 | resources: 61 | - prometheusrules 62 | - servicemonitors 63 | verbs: 64 | - create 65 | - delete 66 | - get 67 | - list 68 | - patch 69 | - update 70 | - watch 71 | - apiGroups: 72 | - policy 73 | resources: 74 | - poddisruptionbudgets 75 | verbs: 76 | - create 77 | - delete 78 | - get 79 | - list 80 | - patch 81 | - update 82 | - watch 83 | - apiGroups: 84 | - zk.cache.ghostbaby.io 85 | resources: 86 | - workloads 87 | verbs: 88 | - create 89 | - delete 90 | - get 91 | - list 92 | - patch 93 | - update 94 | - watch 95 | - apiGroups: 96 | - zk.cache.ghostbaby.io 97 | resources: 98 | - workloads/status 99 | verbs: 100 | - get 101 | - patch 102 | - update 103 | -------------------------------------------------------------------------------- /controllers/workload/workload.go: -------------------------------------------------------------------------------- 1 | package workload 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/prometheus" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 9 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 10 | 11 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/observer" 12 | 13 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 14 | 15 | "github.com/ghostbaby/zookeeper-operator/controllers/k8s" 16 | "github.com/go-logr/logr" 17 | "k8s.io/apimachinery/pkg/runtime" 18 | "k8s.io/client-go/tools/record" 19 | ) 20 | 21 | // ReconcileWorkload implement the Reconciler interface and lcm.Controller interface. 22 | type ReconcileWorkload struct { 23 | Getter 24 | Workload *cachev1alpha1.Workload 25 | CTX context.Context 26 | Client k8s.Client 27 | Recorder record.EventRecorder 28 | Log logr.Logger 29 | DClient k8s.DClient 30 | Scheme *runtime.Scheme 31 | Observers *observer.Manager 32 | Monitor *prometheus.GenericClientset 33 | Labels map[string]string 34 | ZKClient *zk.BaseClient 35 | ObservedState *observer.State 36 | Finalizers finalizer.Handler 37 | } 38 | 39 | func (w *ReconcileWorkload) Reconcile() error { 40 | w.Client.WithContext(w.CTX) 41 | option := w.GetOptions() 42 | 43 | if err := w.ProvisionWorkload(w.CTX, w.Workload, option).Reconcile(); err != nil { 44 | return err 45 | } 46 | 47 | if !w.Workload.GetDeletionTimestamp().IsZero() { 48 | return nil 49 | } 50 | 51 | if err := w.ScaleWorkload(w.CTX, w.Workload, option).Reconcile(); err != nil { 52 | return err 53 | } 54 | 55 | if err := w.RolloutWorkload(w.CTX, w.Workload, option).Reconcile(); err != nil { 56 | return err 57 | } 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /controllers/workload/provision/provision.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/prometheus" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 9 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 10 | 11 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/observer" 12 | 13 | appsv1 "k8s.io/api/apps/v1" 14 | 15 | "k8s.io/apimachinery/pkg/runtime" 16 | 17 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 18 | "github.com/ghostbaby/zookeeper-operator/controllers/k8s" 19 | "github.com/go-logr/logr" 20 | "k8s.io/client-go/tools/record" 21 | ) 22 | 23 | type Provision struct { 24 | Workload *cachev1alpha1.Workload 25 | CTX context.Context 26 | Client k8s.Client 27 | Recorder record.EventRecorder 28 | Log logr.Logger 29 | Labels map[string]string 30 | Monitor *prometheus.GenericClientset 31 | Scheme *runtime.Scheme 32 | ExpectSts *appsv1.StatefulSet 33 | ActualSts *appsv1.StatefulSet 34 | Observers *observer.Manager 35 | ZKClient *zk.BaseClient 36 | ObservedState *observer.State 37 | Finalizers finalizer.Handler 38 | } 39 | 40 | func (p *Provision) Reconcile() error { 41 | 42 | if p.Workload.GetDeletionTimestamp().IsZero() { 43 | 44 | if err := p.ProvisionStatefulset(); err != nil { 45 | return err 46 | } 47 | 48 | if err := p.ProvisionConfigMap(); err != nil { 49 | return err 50 | } 51 | 52 | if err := p.ProvisionService(); err != nil { 53 | return err 54 | } 55 | 56 | if err := p.ProvisionMonitor(); err != nil { 57 | return err 58 | } 59 | 60 | if err := p.Observer(); err != nil { 61 | return err 62 | } 63 | } 64 | 65 | if err := p.Finalizers.Handle(p.Workload, p.FinalizersFor(p.Workload)...); err != nil { 66 | return err 67 | } 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /agent/g/cfg.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "sync" 8 | 9 | "github.com/toolkits/file" 10 | ) 11 | 12 | type HttpConfig struct { 13 | Enabled bool `json:"enabled"` 14 | Listen string `json:"listen"` 15 | Backdoor bool `json:"backdoor"` 16 | } 17 | 18 | type GlobalConfig struct { 19 | Debug bool `json:"debug"` 20 | Hostname string `json:"hostname"` 21 | IP string `json:"ip"` 22 | ZkHost string `json:"zkHost"` 23 | ZkPort string `json:"zkPort"` 24 | Http *HttpConfig `json:"http"` 25 | } 26 | 27 | var ( 28 | ConfigFile string 29 | config *GlobalConfig 30 | lock = new(sync.RWMutex) 31 | ) 32 | 33 | func Config() *GlobalConfig { 34 | lock.RLock() 35 | defer lock.RUnlock() 36 | return config 37 | } 38 | 39 | func Hostname() (string, error) { 40 | hostname := Config().Hostname 41 | if hostname != "" { 42 | return hostname, nil 43 | } 44 | 45 | if os.Getenv("FALCON_ENDPOINT") != "" { 46 | hostname = os.Getenv("FALCON_ENDPOINT") 47 | return hostname, nil 48 | } 49 | 50 | hostname, err := os.Hostname() 51 | if err != nil { 52 | log.Println("ERROR: os.Hostname() fail", err) 53 | } 54 | return hostname, err 55 | } 56 | 57 | func ParseConfig(cfg string) { 58 | if cfg == "" { 59 | log.Fatalln("use -c to specify configuration file") 60 | } 61 | 62 | if !file.IsExist(cfg) { 63 | log.Fatalln("config file:", cfg, "is not existent. maybe you need `mv cfg.example.json cfg.json`") 64 | } 65 | 66 | ConfigFile = cfg 67 | 68 | configContent, err := file.ToTrimString(cfg) 69 | if err != nil { 70 | log.Fatalln("read config file:", cfg, "fail:", err) 71 | } 72 | 73 | var c GlobalConfig 74 | err = json.Unmarshal([]byte(configContent), &c) 75 | if err != nil { 76 | log.Fatalln("parse config file:", cfg, "fail:", err) 77 | } 78 | 79 | lock.Lock() 80 | defer lock.Unlock() 81 | 82 | config = &c 83 | 84 | log.Println("read config file:", cfg, "successfully") 85 | } 86 | -------------------------------------------------------------------------------- /agent/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= 4 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 5 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 6 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= 10 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= 11 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 12 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 13 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 14 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 15 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 16 | github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07 h1:d/VUIMNTk65Xz69htmRPNfjypq2uNRqVsymcXQu6kKk= 17 | github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07/go.mod h1:FbXpUxsx5in7z/OrWFDdhYetOy3/VGIJsVHN9G7RUPA= 18 | github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf h1:1/LnhAvvotcSWDl1ntwUePzLXcyHjAzZ0Ih0F+kKGZU= 19 | github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf/go.mod h1:GejnAYmB2Pr/2fWKp7OGdd6MzuXvRwClmdQAnvnr++I= 20 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 21 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 22 | -------------------------------------------------------------------------------- /controllers/workload/common/observer/watch.go: -------------------------------------------------------------------------------- 1 | package observer 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/types" 6 | "sigs.k8s.io/controller-runtime/pkg/event" 7 | "sigs.k8s.io/controller-runtime/pkg/source" 8 | ) 9 | 10 | // WatchClusterHealthChange returns a Source fed with generic events targeting clusters 11 | // whose health has changed between 2 observations. 12 | // Aimed to be used for triggering a reconciliation. 13 | func WatchClusterHealthChange(m *Manager) *source.Channel { 14 | evtChan := make(chan event.GenericEvent) 15 | m.AddObservationListener(healthChangeListener(evtChan)) 16 | return &source.Channel{ 17 | // Each event in Source will be consumed and turned into 18 | // a reconciliation request. 19 | Source: evtChan, 20 | // DestBufferSize is kept at the default value (1024). 21 | // This means we can enqueue a maximum of 1024 requests 22 | // before blocking observers from moving on. 23 | } 24 | } 25 | 26 | // healthChangeListener returns an OnObservation listener that feeds a generic 27 | // event when a cluster's observed health has changed. 28 | func healthChangeListener(reconciliation chan event.GenericEvent) OnObservation { 29 | return func(cluster types.NamespacedName, previous State, new State) { 30 | // no-op if health hasn't change 31 | if !hasHealthChanged(previous, new) { 32 | return 33 | } 34 | 35 | // trigger a reconciliation event for that cluster 36 | evt := event.GenericEvent{ 37 | Meta: &metav1.ObjectMeta{ 38 | Namespace: cluster.Namespace, 39 | Name: cluster.Name, 40 | }, 41 | } 42 | reconciliation <- evt 43 | } 44 | } 45 | 46 | // hasHealthChanged returns true if previous and new contain different health. 47 | func hasHealthChanged(previous State, new State) bool { 48 | switch { 49 | // both nil 50 | case previous.ClusterStats == nil && new.ClusterStats == nil: 51 | return false 52 | // both equal 53 | case previous.ClusterStats != nil && new.ClusterStats != nil && 54 | previous.ClusterStats.LeaderNode == new.ClusterStats.LeaderNode: 55 | return false 56 | // else: different 57 | default: 58 | return true 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /controllers/getter.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/prometheus" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 9 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 10 | 11 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/observer" 12 | 13 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 14 | "github.com/ghostbaby/zookeeper-operator/controllers/k8s" 15 | w "github.com/ghostbaby/zookeeper-operator/controllers/workload" 16 | "github.com/go-logr/logr" 17 | "k8s.io/apimachinery/pkg/runtime" 18 | "k8s.io/client-go/tools/record" 19 | ) 20 | 21 | type Reconciler interface { 22 | // Reconcile the dependent service. 23 | Reconcile() error 24 | } 25 | 26 | type ServiceGetter interface { 27 | // For Workload 28 | Workload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler 29 | } 30 | 31 | type GetOptions struct { 32 | Client k8s.Client 33 | Recorder record.EventRecorder 34 | Log logr.Logger 35 | DClient k8s.DClient 36 | Scheme *runtime.Scheme 37 | Labels map[string]string 38 | Monitor *prometheus.GenericClientset 39 | Observers *observer.Manager 40 | ZKClient *zk.BaseClient 41 | ObservedState *observer.State 42 | Finalizers finalizer.Handler 43 | } 44 | 45 | type ServiceGetterImpl struct { 46 | } 47 | 48 | func (impl *ServiceGetterImpl) Workload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler { 49 | return &w.ReconcileWorkload{ 50 | Workload: workload, 51 | Client: options.Client, 52 | Recorder: options.Recorder, 53 | Log: options.Log, 54 | DClient: options.DClient, 55 | Scheme: options.Scheme, 56 | CTX: ctx, 57 | Labels: options.Labels, 58 | Getter: &w.GetterImpl{}, 59 | Observers: options.Observers, 60 | ZKClient: options.ZKClient, 61 | ObservedState: options.ObservedState, 62 | Finalizers: options.Finalizers, 63 | Monitor: options.Monitor, 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /controllers/workload/provision/cm.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 8 | 9 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/cm" 10 | 11 | corev1 "k8s.io/api/core/v1" 12 | "k8s.io/apimachinery/pkg/api/errors" 13 | "k8s.io/apimachinery/pkg/types" 14 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 15 | ) 16 | 17 | func (p *Provision) ProvisionConfigMap() error { 18 | namespace := p.Workload.Namespace 19 | 20 | cm := cm.NewCM(p.Workload, p.Labels, p.ExpectSts, p.ActualSts) 21 | 22 | scs, err := cm.GenerateConfigMap() 23 | if err != nil { 24 | return err 25 | } 26 | 27 | for _, sc := range scs { 28 | cm := &corev1.ConfigMap{} 29 | if err := controllerutil.SetControllerReference(p.Workload, sc, p.Scheme); err != nil { 30 | p.Log.Error(err, "Set OwnerReference fail.", "namespace", cm.Namespace, "name", cm.Name) 31 | return err 32 | } 33 | 34 | err = p.Client.Get(types.NamespacedName{Name: sc.Name, Namespace: namespace}, cm) 35 | if err != nil && errors.IsNotFound(err) { 36 | p.Log.Info("Creating ZooKeeper ConfigMap .") 37 | err = p.Client.Create(sc) 38 | if err != nil { 39 | p.Log.Error(err, "Create ConfigMap fail.", "namespace", cm.Namespace, "name", cm.Name) 40 | return err 41 | } 42 | msg := fmt.Sprintf(model.MessageZooKeeperConfigMap, sc.Name) 43 | p.Recorder.Event(p.Workload, corev1.EventTypeNormal, model.ZooKeeperConfigMap, msg) 44 | } else if err != nil { 45 | p.Log.Error(err, "Create ConfigMap fail.", "namespace", cm.Namespace, "name", cm.Name) 46 | return err 47 | } else { 48 | if !reflect.DeepEqual(sc.Data, cm.Data) { 49 | cm.Data = sc.Data 50 | msg := fmt.Sprintf(model.UpdateMessageZooKeeperConfigMap, sc.Name) 51 | p.Recorder.Event(p.Workload, corev1.EventTypeNormal, model.ZooKeeperConfigMap, msg) 52 | p.Log.Info("Updating ConfigMap .", "namespace", cm.Namespace, "name", cm.Name) 53 | err = p.Client.Update(cm) 54 | if err != nil { 55 | p.Log.Error(err, "Update ConfigMap fail.", "namespace", cm.Namespace, "name", cm.Name) 56 | return err 57 | } 58 | } 59 | } 60 | 61 | } 62 | 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /config/samples/oam/oam.md: -------------------------------------------------------------------------------- 1 | # zookeeper-operator access OAM framework 2 | 3 | ## Prerequisite 4 | 5 | Make sure [`OAM runtime`](https://github.com/crossplane/oam-kubernetes-runtime/blob/master/README.md) was installed and started. 6 | 7 | ## Install Zookeeper-operator 8 | 9 | 10 | Step 1. Modify the Makefile file and replace `CRD_OPTIONS ?= "crd:crdVersions=v1,trivialVersions=false"` with CRD_OPTIONS ?= "crd:trivialVersions=true" to generate apiextensions.k8s.io/v1 version CRD. 11 | 12 | Step 2. Use `make install` to install zookeeper-operator CRD. 13 | 14 | Step 3. Use `make deploy` to install zookeeper-operator Controller. 15 | 16 | ## Registry Zookeeper-operator to OAM Workload 17 | 18 | ```yaml 19 | apiVersion: core.oam.dev/v1alpha2 20 | kind: WorkloadDefinition 21 | metadata: 22 | name: workloads.zk.cache.ghostbaby.io 23 | spec: 24 | definitionRef: 25 | name: workloads.zk.cache.ghostbaby.io 26 | ``` 27 | 28 | ## Create Zookeeper-operator Component 29 | ```yaml 30 | apiVersion: core.oam.dev/v1alpha2 31 | kind: Component 32 | metadata: 33 | name: zk-component 34 | spec: 35 | workload: 36 | apiVersion: zk.cache.ghostbaby.io/v1alpha1 37 | kind: Workload 38 | spec: 39 | version: v3.5.6 40 | cluster: 41 | name: test 42 | resources: 43 | requests: 44 | cpu: 100m 45 | memory: 500Mi 46 | exporter: 47 | exporter: true 48 | exporterImage: ghostbaby/zookeeper_exporter 49 | exporterVersion: v3.5.6 50 | disableExporterProbes: false 51 | parameters: 52 | - name: name 53 | fieldPaths: 54 | - metadata.name 55 | ``` 56 | 57 | ## Create Zookeeper Cluster and bind ManualScalerTrait 58 | ```yaml 59 | apiVersion: core.oam.dev/v1alpha2 60 | kind: ApplicationConfiguration 61 | metadata: 62 | name: zk-appconfig 63 | spec: 64 | components: 65 | - componentName: zk-component 66 | parameterValues: 67 | - name: name 68 | value: ghostbaby 69 | traits: 70 | - trait: 71 | apiVersion: core.oam.dev/v1alpha2 72 | kind: ManualScalerTrait 73 | metadata: 74 | name: zk-appconfig-trait 75 | spec: 76 | replicaCount: 3 77 | ``` -------------------------------------------------------------------------------- /controllers/k8s/new.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/apimachinery/pkg/api/meta" 7 | "k8s.io/client-go/discovery" 8 | "k8s.io/client-go/rest" 9 | "k8s.io/client-go/tools/clientcmd" 10 | 11 | k8sruntime "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/cli-runtime/pkg/genericclioptions" 13 | "k8s.io/cli-runtime/pkg/resource" 14 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 15 | "sigs.k8s.io/controller-runtime/pkg/client" 16 | ) 17 | 18 | func NewKubeClient() (client.Client, *rest.Config, error) { 19 | 20 | scheme := k8sruntime.NewScheme() 21 | _ = clientgoscheme.AddToScheme(scheme) 22 | 23 | kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() 24 | f := NewFactory(kubeConfigFlags) 25 | restConf, err := f.ToRESTConfig() 26 | if err != nil { 27 | fmt.Println("get kubeconfig err", err) 28 | return nil, nil, err 29 | } 30 | client, err := client.New(restConf, client.Options{Scheme: scheme}) 31 | if err != nil { 32 | fmt.Println("create client from kubeconfig err", err) 33 | return nil, nil, err 34 | } 35 | return client, restConf, nil 36 | } 37 | 38 | type Factory interface { 39 | genericclioptions.RESTClientGetter 40 | NewBuilder() *resource.Builder 41 | } 42 | 43 | type factoryImpl struct { 44 | clientGetter genericclioptions.RESTClientGetter 45 | } 46 | 47 | func (f *factoryImpl) ToRESTConfig() (*rest.Config, error) { 48 | return f.clientGetter.ToRESTConfig() 49 | } 50 | 51 | func (f *factoryImpl) ToRESTMapper() (meta.RESTMapper, error) { 52 | return f.clientGetter.ToRESTMapper() 53 | } 54 | 55 | func (f *factoryImpl) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { 56 | return f.clientGetter.ToDiscoveryClient() 57 | } 58 | 59 | func (f *factoryImpl) ToRawKubeConfigLoader() clientcmd.ClientConfig { 60 | return f.clientGetter.ToRawKubeConfigLoader() 61 | } 62 | 63 | // NewBuilder returns a new resource builder for structured api objects. 64 | func (f *factoryImpl) NewBuilder() *resource.Builder { 65 | return resource.NewBuilder(f.clientGetter) 66 | } 67 | 68 | func NewFactory(clientGetter genericclioptions.RESTClientGetter) Factory { 69 | if clientGetter == nil { 70 | panic("attempt to instantiate client_access_factory with nil clientGetter") 71 | } 72 | 73 | f := &factoryImpl{ 74 | clientGetter: clientGetter, 75 | } 76 | 77 | return f 78 | } 79 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= ghostbaby/zookeeper-operator:dev 4 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 5 | CRD_OPTIONS ?= "crd:trivialVersions=true" 6 | #CRD_OPTIONS ?= "crd:crdVersions=v1,trivialVersions=false" 7 | 8 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 9 | ifeq (,$(shell go env GOBIN)) 10 | GOBIN=$(shell go env GOPATH)/bin 11 | else 12 | GOBIN=$(shell go env GOBIN) 13 | endif 14 | 15 | all: manager 16 | 17 | # Run tests 18 | test: generate fmt vet manifests 19 | go test ./... -coverprofile cover.out 20 | 21 | # Build manager binary 22 | manager: generate fmt vet 23 | go build -o bin/manager main.go 24 | 25 | # Run against the configured Kubernetes cluster in ~/.kube/config 26 | run: generate fmt vet manifests 27 | go run ./main.go 28 | 29 | # Install CRDs into a cluster 30 | install: manifests 31 | kustomize build config/crd | kubectl apply -f - 32 | 33 | # Uninstall CRDs from a cluster 34 | uninstall: manifests 35 | kustomize build config/crd | kubectl delete -f - 36 | 37 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 38 | deploy: manifests 39 | cd config/manager && kustomize edit set image controller=${IMG} 40 | kustomize build config/default | kubectl apply -f - 41 | 42 | # Generate manifests e.g. CRD, RBAC etc. 43 | manifests: controller-gen 44 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases 45 | 46 | # Run go fmt against code 47 | fmt: 48 | go fmt ./... 49 | 50 | # Run go vet against code 51 | vet: 52 | go vet ./... 53 | 54 | # Generate code 55 | generate: controller-gen 56 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 57 | 58 | # Build the docker image 59 | docker-build: test 60 | docker build . -t ${IMG} 61 | 62 | # Push the docker image 63 | docker-push: 64 | docker push ${IMG} 65 | 66 | # find or download controller-gen 67 | # download controller-gen if necessary 68 | controller-gen: 69 | ifeq (, $(shell which controller-gen)) 70 | @{ \ 71 | set -e ;\ 72 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ 73 | cd $$CONTROLLER_GEN_TMP_DIR ;\ 74 | go mod init tmp ;\ 75 | go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5 ;\ 76 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ 77 | } 78 | CONTROLLER_GEN=$(GOBIN)/controller-gen 79 | else 80 | CONTROLLER_GEN=$(shell which controller-gen) 81 | endif 82 | 83 | -------------------------------------------------------------------------------- /controllers/workload/common/sts/initcontainer.go: -------------------------------------------------------------------------------- 1 | package sts 2 | 3 | import ( 4 | "path" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/cm" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/volume" 9 | corev1 "k8s.io/api/core/v1" 10 | ) 11 | 12 | var ( 13 | // PodDownwardEnvVars inject the runtime Pod Name and IP as environment variables. 14 | PodDownwardEnvVars = []corev1.EnvVar{ 15 | {Name: EnvPodIP, Value: "", ValueFrom: &corev1.EnvVarSource{ 16 | FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "status.podIP"}, 17 | }}, 18 | {Name: EnvPodName, Value: "", ValueFrom: &corev1.EnvVarSource{ 19 | FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}, 20 | }}, 21 | } 22 | ) 23 | 24 | func NewPrepareFSInitContainer(imageName string) (corev1.Container, error) { 25 | // we mount the certificates to a location outside of the default config directory because the prepare-fs script 26 | // will attempt to move all the files under the configuration directory to a different volume, and it should not 27 | // be attempting to move files from this secret volume mount (any attempt to do so will be logged as errors). 28 | 29 | scriptsVolume := corev1.VolumeMount{ 30 | Name: volume.ScriptsVolumeName, 31 | MountPath: volume.ScriptsVolumeMountPath, 32 | ReadOnly: true, 33 | } 34 | 35 | privileged := false 36 | container := corev1.Container{ 37 | Image: imageName, 38 | ImagePullPolicy: corev1.PullAlways, 39 | Name: PrepareFilesystemContainerName, 40 | SecurityContext: &corev1.SecurityContext{ 41 | Privileged: &privileged, 42 | }, 43 | Env: PodDownwardEnvVars, 44 | Command: []string{"bash", "-c", path.Join(volume.ScriptsVolumeMountPath, cm.PrepareFsScriptConfigKey)}, 45 | VolumeMounts: append( 46 | InitContainerVolumeMounts(cm.PluginVolumes), 47 | scriptsVolume, 48 | volume.DefaultDataVolumeMount, 49 | volume.DefaultLogsVolumeMount, 50 | ), 51 | } 52 | 53 | return container, nil 54 | } 55 | 56 | func InitContainerVolumeMount(v cm.SharedVolume) corev1.VolumeMount { 57 | return corev1.VolumeMount{ 58 | MountPath: v.InitContainerMountPath, 59 | Name: v.Name, 60 | } 61 | } 62 | 63 | func InitContainerVolumeMounts(v cm.SharedVolumeArray) []corev1.VolumeMount { 64 | mounts := make([]corev1.VolumeMount, len(v.Array)) 65 | for i, v := range v.Array { 66 | mounts[i] = InitContainerVolumeMount(v) 67 | } 68 | return mounts 69 | } 70 | -------------------------------------------------------------------------------- /controllers/workload/common/cm/new.go: -------------------------------------------------------------------------------- 1 | package cm 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | ) 12 | 13 | type ZkConfigMap struct { 14 | Data map[string]string `json:"data"` 15 | Name string `json:"name"` 16 | } 17 | 18 | func (c *CM) GenerateConfigMap() ([]*corev1.ConfigMap, error) { 19 | name := c.Workload.Name 20 | namespace := c.Workload.Namespace 21 | var list []*ZkConfigMap 22 | 23 | //生成agent启动配置文件 24 | agent := GenZkAgentConfig() 25 | 26 | agentData := map[string]string{ 27 | AgentConfigKey: agent, 28 | } 29 | 30 | list = append(list, &ZkConfigMap{ 31 | Data: agentData, 32 | Name: genConfigMapName(name, AgentVolumeName), 33 | }) 34 | 35 | //生成初始化脚本 36 | fsScript, err := RenderPrepareFsScript() 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | scriptData := map[string]string{ 42 | PrepareFsScriptConfigKey: fsScript, 43 | } 44 | 45 | list = append(list, &ZkConfigMap{ 46 | Data: scriptData, 47 | Name: genConfigMapName(name, ScriptsVolumeName), 48 | }) 49 | 50 | //生成zookeeper启动配置文件 51 | config := GenZkConfig() 52 | 53 | configData := map[string]string{ 54 | ConfigFileName: config, 55 | } 56 | 57 | list = append(list, &ZkConfigMap{ 58 | Data: configData, 59 | Name: genConfigMapName(name, ConfigVolumeName), 60 | }) 61 | 62 | //生成动态配置文件 63 | podNames := utils.PodNames(*c.ExpectSts) 64 | podIpArray := utils.GetPodIp(c.Workload, podNames) 65 | 66 | var hosts string 67 | if podIpArray != nil { 68 | hosts = strings.Join(podIpArray, "\n") 69 | } 70 | 71 | dynamicConfigData := map[string]string{ 72 | DynamicConfigFile: hosts, 73 | } 74 | 75 | list = append(list, &ZkConfigMap{ 76 | Data: dynamicConfigData, 77 | Name: genConfigMapName(name, DynamicConfigFileVolumeName), 78 | }) 79 | 80 | cmList := GenConfigMap(list, namespace, c.Labels) 81 | 82 | return cmList, nil 83 | } 84 | 85 | func GenConfigMap(listArray []*ZkConfigMap, namespace string, labels map[string]string) []*corev1.ConfigMap { 86 | var cmList []*corev1.ConfigMap 87 | for _, data := range listArray { 88 | cm := &corev1.ConfigMap{ 89 | ObjectMeta: metav1.ObjectMeta{ 90 | Name: data.Name, 91 | Namespace: namespace, 92 | Labels: labels, 93 | }, 94 | Data: data.Data, 95 | } 96 | cmList = append(cmList, cm) 97 | } 98 | return cmList 99 | } 100 | 101 | func genConfigMapName(name, cmType string) string { 102 | return fmt.Sprintf("%s-%s", name, cmType) 103 | } 104 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: zookeeper-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: zookeeper-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | - ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | - ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | patchesStrategicMerge: 28 | # Protect the /metrics endpoint by putting it behind auth. 29 | # If you want your controller-manager to expose the /metrics 30 | # endpoint w/o any authn/z, please comment the following line. 31 | - manager_auth_proxy_patch.yaml 32 | 33 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 34 | # crd/kustomization.yaml 35 | - manager_webhook_patch.yaml 36 | 37 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 38 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 39 | # 'CERTMANAGER' needs to be enabled to use ca injection 40 | - webhookcainjection_patch.yaml 41 | 42 | # the following config is for teaching kustomize how to do var substitution 43 | vars: 44 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 45 | - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 46 | objref: 47 | kind: Certificate 48 | group: cert-manager.io 49 | version: v1alpha2 50 | name: serving-cert # this name should match the one in certificate.yaml 51 | fieldref: 52 | fieldpath: metadata.namespace 53 | - name: CERTIFICATE_NAME 54 | objref: 55 | kind: Certificate 56 | group: cert-manager.io 57 | version: v1alpha2 58 | name: serving-cert # this name should match the one in certificate.yaml 59 | - name: SERVICE_NAMESPACE # namespace of the service 60 | objref: 61 | kind: Service 62 | version: v1 63 | name: webhook-service 64 | fieldref: 65 | fieldpath: metadata.namespace 66 | - name: SERVICE_NAME 67 | objref: 68 | kind: Service 69 | version: v1 70 | name: webhook-service 71 | -------------------------------------------------------------------------------- /controllers/workload/common/sts/resource.go: -------------------------------------------------------------------------------- 1 | package sts 2 | 3 | import ( 4 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 5 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 6 | appsv1 "k8s.io/api/apps/v1" 7 | corev1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/api/resource" 9 | ) 10 | 11 | func getResources(w *cachev1alpha1.Workload) corev1.ResourceRequirements { 12 | 13 | reqCpu := w.Spec.Cluster.Resources.Requests.CPU 14 | reqMem := w.Spec.Cluster.Resources.Requests.Memory 15 | limCpu := w.Spec.Cluster.Resources.Limits.CPU 16 | limMem := w.Spec.Cluster.Resources.Limits.Memory 17 | 18 | return corev1.ResourceRequirements{ 19 | Requests: getRequests(reqCpu, reqMem), 20 | Limits: getLimits(limCpu, limMem), 21 | } 22 | } 23 | 24 | func getLimits(limCpu string, limitMem string) corev1.ResourceList { 25 | return generateResourceList(limCpu, limitMem) 26 | } 27 | 28 | func getRequests(reqCpu string, reqMem string) corev1.ResourceList { 29 | return generateResourceList(reqCpu, reqMem) 30 | } 31 | 32 | func generateResourceList(cpu string, memory string) corev1.ResourceList { 33 | resources := corev1.ResourceList{} 34 | if cpu != "" { 35 | resources[corev1.ResourceCPU], _ = resource.ParseQuantity(cpu) 36 | } 37 | if memory != "" { 38 | resources[corev1.ResourceMemory], _ = resource.ParseQuantity(memory) 39 | } 40 | return resources 41 | } 42 | 43 | func getStsResource(sts *appsv1.StatefulSet) *cachev1alpha1.ZkResource { 44 | var requestCpu, requestMem *resource.Quantity 45 | var limitCpu, limitMem *resource.Quantity 46 | for _, container := range sts.Spec.Template.Spec.Containers { 47 | if container.Name == model.RoleName { 48 | requestCpu = container.Resources.Requests.Cpu() 49 | requestMem = container.Resources.Requests.Memory() 50 | 51 | limitCpu = container.Resources.Limits.Cpu() 52 | limitMem = container.Resources.Limits.Memory() 53 | } else { 54 | continue 55 | } 56 | } 57 | return &cachev1alpha1.ZkResource{ 58 | RequestCpu: requestCpu, 59 | RequestMem: requestMem, 60 | LimitCpu: limitCpu, 61 | LimitMem: limitMem, 62 | } 63 | } 64 | 65 | func IsUpgradeStsResource(expectSts *appsv1.StatefulSet, actualSts *appsv1.StatefulSet) bool { 66 | expectEsResource := getStsResource(expectSts) 67 | actualEsResource := getStsResource(actualSts) 68 | 69 | if expectEsResource.RequestCpu.String() != actualEsResource.RequestCpu.String() || 70 | expectEsResource.RequestMem.String() != actualEsResource.RequestMem.String() || 71 | expectEsResource.LimitCpu.String() != actualEsResource.LimitCpu.String() || 72 | expectEsResource.LimitMem.String() != actualEsResource.LimitMem.String() { 73 | return true 74 | } else { 75 | return false 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "path/filepath" 21 | "testing" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/rest" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/envtest" 29 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 30 | logf "sigs.k8s.io/controller-runtime/pkg/log" 31 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 32 | 33 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 34 | // +kubebuilder:scaffold:imports 35 | ) 36 | 37 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 38 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 39 | 40 | var cfg *rest.Config 41 | var k8sClient client.Client 42 | var testEnv *envtest.Environment 43 | 44 | func TestAPIs(t *testing.T) { 45 | RegisterFailHandler(Fail) 46 | 47 | RunSpecsWithDefaultAndCustomReporters(t, 48 | "Controller Suite", 49 | []Reporter{printer.NewlineReporter{}}) 50 | } 51 | 52 | var _ = BeforeSuite(func(done Done) { 53 | logf.SetLogger(zap.LoggerTo(GinkgoWriter, true)) 54 | 55 | By("bootstrapping test environment") 56 | testEnv = &envtest.Environment{ 57 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 58 | } 59 | 60 | var err error 61 | cfg, err = testEnv.Start() 62 | Expect(err).ToNot(HaveOccurred()) 63 | Expect(cfg).ToNot(BeNil()) 64 | 65 | err = cachev1alpha1.AddToScheme(scheme.Scheme) 66 | Expect(err).NotTo(HaveOccurred()) 67 | 68 | err = cachev1alpha1.AddToScheme(scheme.Scheme) 69 | Expect(err).NotTo(HaveOccurred()) 70 | 71 | err = cachev1alpha1.AddToScheme(scheme.Scheme) 72 | Expect(err).NotTo(HaveOccurred()) 73 | 74 | err = cachev1alpha1.AddToScheme(scheme.Scheme) 75 | Expect(err).NotTo(HaveOccurred()) 76 | 77 | err = cachev1alpha1.AddToScheme(scheme.Scheme) 78 | Expect(err).NotTo(HaveOccurred()) 79 | 80 | // +kubebuilder:scaffold:scheme 81 | 82 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 83 | Expect(err).ToNot(HaveOccurred()) 84 | Expect(k8sClient).ToNot(BeNil()) 85 | 86 | close(done) 87 | }, 60) 88 | 89 | var _ = AfterSuite(func() { 90 | By("tearing down the test environment") 91 | err := testEnv.Stop() 92 | Expect(err).ToNot(HaveOccurred()) 93 | }) 94 | -------------------------------------------------------------------------------- /controllers/workload/model/const.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | const ( 4 | AppLabel = "zookeeper" 5 | RoleName = "zookeeper" 6 | ContainerName = "zookeeper" 7 | DefaultImageRepository string = "ghostbaby/zookeeper" 8 | ) 9 | 10 | const ( 11 | ZooKeeperStatefulset = "ZooKeeperStatefulset" 12 | ZooKeeperService = "ZooKeeperService" 13 | ZooKeeperConfigMap = "ZooKeeperConfigMap" 14 | ZooKeeperServiceMonitor = "ZooKeeperServiceMonitor" 15 | ZooKeeperPodDisruptionBudget = "ZooKeeperPodDisruptionBudget" 16 | ZooKeeperUserSecret = "ZooKeeperUserSecret" 17 | ZooKeeperKeySecret = "ZooKeeperKeySecret" 18 | ZooKeeperPrometheusRules = "ZooKeeperPrometheusRules" 19 | 20 | ZooKeeperDownScaling = "ZooKeeperDownScaling" 21 | ZooKeeperUpScaling = "ZooKeeperUpScaling" 22 | 23 | MessageZooKeeperStatefulset = "ZooKeeper Statefulset %s already created." 24 | MessageZooKeeperService = "ZooKeeper Service %s already created." 25 | MessageZooKeeperConfigMap = "ZooKeeper ConfigMap %s already created." 26 | MessageZooKeeperServiceMonitor = "ZooKeeper ServiceMonitor %s already created." 27 | MessageZooKeeperPrometheusRules = "ZooKeeper PrometheusRules %s already created." 28 | MessageZooKeeperPodDisruptionBudget = "ZooKeeper PodDisruptionBudget %s already created." 29 | MessageZooKeeperUserSecret = "ZooKeeper User Secret %s already created." 30 | MessageZooKeeperKeySecret = "ZooKeeper Key Secret %s already created." 31 | 32 | UpdateMessageZooKeeperStatefulset = "ZooKeeper Statefulset %s already update." 33 | UpdateMessageZooKeeperConfigMap = "ZooKeeper ConfigMap %s already update." 34 | 35 | MessageZooKeeperDownScaling = "ZooKeeper downscale from %d to %d" 36 | MessageZooKeeperUpScaling = "ZooKeeper upscale from %d to %d" 37 | ) 38 | 39 | const ( 40 | ExporterPort = 9114 41 | ExporterPortName = "http-metrics" 42 | ExporterContainerName = "zk-exporter" 43 | ExporterDefaultRequestCPU = "25m" 44 | ExporterDefaultLimitCPU = "50m" 45 | ExporterDefaultRequestMemory = "50Mi" 46 | ExporterDefaultLimitMemory = "100Mi" 47 | ) 48 | 49 | const ( 50 | AgentPort = 1988 51 | AgentPortName = "http-agent" 52 | AgentContainerName = "zk-agent" 53 | AgentDefaultRequestCPU = "25m" 54 | AgentDefaultLimitCPU = "50m" 55 | AgentDefaultRequestMemory = "100Mi" 56 | AgentDefaultLimitMemory = "200Mi" 57 | ) 58 | 59 | const ( 60 | ClientPort = 2181 61 | ServerPort = 2888 62 | LeaderElectionPort = 3888 63 | AgentHTTPPort = 1988 64 | 65 | ClusterIPServiceType = "ClusterIP" 66 | HeadlessServiceType = "Headless" 67 | ) 68 | 69 | const ( 70 | ServiceMonitorInterval = "30s" 71 | ServiceMonitorPort = "http-metrics" 72 | ServiceMonitorCrdName = "servicemonitors.monitoring.coreos.com" 73 | PrometheusRulesCrdName = "prometheusrules.monitoring.coreos.com" 74 | StorageLowAlertName = "存储空间低" 75 | ) 76 | -------------------------------------------------------------------------------- /controllers/workload/common/zk/base.go: -------------------------------------------------------------------------------- 1 | package zk 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "io" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 13 | ) 14 | 15 | type BaseClient struct { 16 | Endpoints []string 17 | HTTP *http.Client 18 | Endpoint string 19 | Transport *http.Transport 20 | } 21 | 22 | func (c *BaseClient) Get(ctx context.Context, pathWithQuery string, out interface{}) error { 23 | return c.request(ctx, http.MethodGet, pathWithQuery, nil, out) 24 | } 25 | 26 | func (c *BaseClient) Post(ctx context.Context, pathWithQuery string, in, out interface{}) error { 27 | return c.request(ctx, http.MethodPost, pathWithQuery, in, out) 28 | } 29 | 30 | func (c *BaseClient) request( 31 | ctx context.Context, 32 | method string, 33 | pathWithQuery string, 34 | requestObj, 35 | responseObj interface{}, 36 | ) error { 37 | var body io.Reader = http.NoBody 38 | if requestObj != nil { 39 | outData, err := json.Marshal(requestObj) 40 | if err != nil { 41 | return err 42 | } 43 | body = bytes.NewBuffer(outData) 44 | } 45 | 46 | request, err := http.NewRequest(method, utils.Joins(c.Endpoint, pathWithQuery), body) 47 | if err != nil { 48 | return err 49 | } 50 | resp, err := c.doRequest(ctx, request) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | defer resp.Body.Close() 56 | 57 | if responseObj != nil { 58 | if err := json.NewDecoder(resp.Body).Decode(responseObj); err != nil { 59 | return err 60 | } 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (c *BaseClient) doRequest(context context.Context, request *http.Request) (*http.Response, error) { 67 | withContext := request.WithContext(context) 68 | withContext.Header.Set("Content-Type", "application/json; charset=utf-8") 69 | 70 | response, err := c.HTTP.Do(withContext) 71 | if err != nil { 72 | return response, err 73 | } 74 | err = checkError(response) 75 | return response, err 76 | } 77 | 78 | func checkError(response *http.Response) error { 79 | if response.StatusCode < 200 || response.StatusCode >= 300 { 80 | return errors.New("get zk agent fail pls check") 81 | } 82 | return nil 83 | } 84 | 85 | func (c *BaseClient) Close() { 86 | if c.Transport != nil { 87 | // When the http transport goes out of scope, the underlying goroutines responsible 88 | // for handling keep-alive connections are not closed automatically. 89 | // Since this client gets recreated frequently we would effectively be leaking goroutines. 90 | // Let's make sure this does not happen by closing idle connections. 91 | c.Transport.CloseIdleConnections() 92 | } 93 | } 94 | 95 | func (c *BaseClient) Equal(c2 *BaseClient) bool { 96 | // handle nil case 97 | if c2 == nil && c != nil { 98 | return false 99 | } 100 | 101 | // compare endpoint and user creds 102 | return c.Endpoint == c2.Endpoint 103 | } 104 | 105 | func (c *BaseClient) IsAlive(c2 *BaseClient) bool { 106 | // handle nil case 107 | if c2 == nil && c != nil { 108 | return false 109 | } 110 | 111 | timeoutCtx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) 112 | defer cancel() 113 | ok, _ := c.GetClusterUp(timeoutCtx) 114 | return ok 115 | } 116 | -------------------------------------------------------------------------------- /controllers/workload/common/observer/manager.go: -------------------------------------------------------------------------------- 1 | package observer 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 7 | "k8s.io/apimachinery/pkg/types" 8 | ) 9 | 10 | // Manager for a set of observers 11 | type Manager struct { 12 | observers map[types.NamespacedName]*Observer 13 | listeners []OnObservation // invoked on each observation event 14 | lock sync.RWMutex 15 | settings Settings 16 | } 17 | 18 | // NewManager returns a new manager 19 | func NewManager(settings Settings) *Manager { 20 | return &Manager{ 21 | lock: sync.RWMutex{}, 22 | settings: settings, 23 | observers: make(map[types.NamespacedName]*Observer), 24 | } 25 | } 26 | 27 | // Observe gets or create a cluster state observer for the given cluster 28 | // In case something has changed in the given zkClient (eg. different caCert), the observer is recreated accordingly 29 | func (m *Manager) Observe(cluster types.NamespacedName, zkClient zk.BaseClient) *Observer { 30 | m.lock.RLock() 31 | observer, exists := m.observers[cluster] 32 | m.lock.RUnlock() 33 | 34 | switch { 35 | case !exists: 36 | return m.createObserver(cluster, zkClient) 37 | //case exists && !observer.zkClient.Equal(&zkClient): 38 | case exists && !observer.zkClient.IsAlive(&zkClient): 39 | log.Info("Replacing observer HTTP client", "namespace", cluster.Namespace, "zk_name", cluster.Name) 40 | m.StopObserving(cluster) 41 | return m.createObserver(cluster, zkClient) 42 | default: 43 | return observer 44 | } 45 | } 46 | 47 | // createObserver creates a new observer according to the given arguments, 48 | // and create/replace its entry in the observers map 49 | func (m *Manager) createObserver(cluster types.NamespacedName, zkClient zk.BaseClient) *Observer { 50 | observer := NewObserver(cluster, zkClient, m.settings, m.notifyListeners) 51 | observer.Start() 52 | m.lock.Lock() 53 | m.observers[cluster] = observer 54 | m.lock.Unlock() 55 | return observer 56 | } 57 | 58 | func (m *Manager) ObservedStateResolver(cluster types.NamespacedName, zkClient zk.BaseClient) State { 59 | return m.Observe(cluster, zkClient).LastState() 60 | } 61 | 62 | // notifyListeners notifies all listeners that an observation occurred. 63 | func (m *Manager) notifyListeners(cluster types.NamespacedName, previousState State, newState State) { 64 | wg := sync.WaitGroup{} 65 | m.lock.Lock() 66 | wg.Add(len(m.listeners)) 67 | // run all listeners in parallel 68 | for _, l := range m.listeners { 69 | go func(f OnObservation) { 70 | defer wg.Done() 71 | f(cluster, previousState, newState) 72 | }(l) 73 | } 74 | // release the lock asap 75 | m.lock.Unlock() 76 | // wait for all listeners to be done 77 | wg.Wait() 78 | } 79 | 80 | // AddObservationListener adds the given listener to the list of listeners notified 81 | // on every observation. 82 | func (m *Manager) AddObservationListener(listener OnObservation) { 83 | m.lock.Lock() 84 | defer m.lock.Unlock() 85 | m.listeners = append(m.listeners, listener) 86 | } 87 | 88 | func (m *Manager) StopObserving(cluster types.NamespacedName) { 89 | m.lock.RLock() 90 | observer, exists := m.observers[cluster] 91 | m.lock.RUnlock() 92 | if !exists { 93 | return 94 | } 95 | observer.Stop() 96 | m.lock.Lock() 97 | delete(m.observers, cluster) 98 | m.lock.Unlock() 99 | } 100 | -------------------------------------------------------------------------------- /controllers/k8s/client.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | // DefaultTimeout is a reasonable timeout to use with the Client. 12 | const DefaultTimeout = 1 * time.Minute 13 | 14 | // WrapClient returns a Client that performs requests within DefaultTimeout. 15 | func WrapClient(ctx context.Context, client client.Client) Client { 16 | return &ClusterClient{ 17 | crClient: client, 18 | ctx: ctx, 19 | } 20 | } 21 | 22 | // Client wraps a controller-runtime client to use a 23 | // default context with a timeout if no context is passed. 24 | type Client interface { 25 | // WithContext returns a client configured to use the provided context on 26 | // subsequent requests, instead of one created from the preconfigured timeout. 27 | WithContext(ctx context.Context) Client 28 | 29 | // Get wraps a controller-runtime client.Get call with a context. 30 | Get(key client.ObjectKey, obj runtime.Object) error 31 | // List wraps a controller-runtime client.List call with a context. 32 | List(opts *client.ListOptions, list runtime.Object) error 33 | // Create wraps a controller-runtime client.Create call with a context. 34 | Create(obj runtime.Object) error 35 | // Delete wraps a controller-runtime client.Delete call with a context. 36 | Delete(obj runtime.Object, opts ...client.DeleteOption) error 37 | // Update wraps a controller-runtime client.Update call with a context. 38 | Update(obj runtime.Object) error 39 | // Patch wraps a controller-runtime client.Patch call with a context. 40 | Patch(obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error 41 | // WriteStatus wraps a controller-runtime client.status().Update() call with a context. 42 | WriteStatus(obj runtime.Object) error 43 | } 44 | 45 | type ClusterClient struct { 46 | crClient client.Client 47 | ctx context.Context 48 | } 49 | 50 | // WithContext returns a client configured to use the provided context on 51 | // subsequent requests, instead of one created from the preconfigured timeout. 52 | func (w *ClusterClient) WithContext(ctx context.Context) Client { 53 | w.ctx = ctx 54 | return w 55 | } 56 | 57 | // Get wraps a controller-runtime client.Get call with a context. 58 | func (w *ClusterClient) Get(key client.ObjectKey, obj runtime.Object) error { 59 | return w.crClient.Get(w.ctx, key, obj) 60 | } 61 | 62 | // List wraps a controller-runtime client.List call with a context. 63 | func (w *ClusterClient) List(opts *client.ListOptions, list runtime.Object) error { 64 | return w.crClient.List(w.ctx, list, opts) 65 | } 66 | 67 | // Create wraps a controller-runtime client.Create call with a context. 68 | func (w *ClusterClient) Create(obj runtime.Object) error { 69 | return w.crClient.Create(w.ctx, obj) 70 | } 71 | 72 | // Update wraps a controller-runtime client.Update call with a context. 73 | func (w *ClusterClient) Update(obj runtime.Object) error { 74 | return w.crClient.Update(w.ctx, obj) 75 | } 76 | 77 | // Delete wraps a controller-runtime client.Delete call with a context. 78 | func (w *ClusterClient) Delete(obj runtime.Object, opts ...client.DeleteOption) error { 79 | return w.crClient.Delete(w.ctx, obj, opts...) 80 | } 81 | 82 | // Patch wraps a controller-runtime client.Patch call with a context. 83 | func (w *ClusterClient) Patch(obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error { 84 | return w.crClient.Patch(w.ctx, obj, patch, opts...) 85 | } 86 | 87 | func (w *ClusterClient) WriteStatus(obj runtime.Object) error { 88 | return w.crClient.Status().Update(w.ctx, obj) 89 | } 90 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at zhuhuijunzhj@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ZooKeeper Operator 2 | [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) 3 | [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/) 4 | [![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://godoc.org/github.com/apache/rocketmq-operator/pkg) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/Ghostbaby/zookeeper-operator)](https://goreportcard.com/report/github.com/Ghostbaby/zookeeper-operator) 6 | [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/Ghostbaby/zookeeper-operator.svg)](http://isitmaintained.com/project/Ghostbaby/zookeeper-operator "Average time to resolve an issue") 7 | [![Percentage of issues still open](http://isitmaintained.com/badge/open/Ghostbaby/zookeeper-operator.svg)](http://isitmaintained.com/project/Ghostbaby/zookeeper-operator "Percentage of issues still open") 8 | 9 | ## Overview 10 | ZooKeeper Operator is to manage ZooKeeper service instances deployed on the Kubernetes cluster. 11 | It is built using the [Kubebuilder SDK](https://kubebuilder.io/), which is part of the [Kubebuilder](https://github.com/kubernetes-sigs/kubebuilder). 12 | 13 | ## Features 14 | 15 | With this operator, you're able to deploy and manage a HA Zookeeper Cluster: 16 | 17 | - [x] Provision a Zookeeper cluster in a scalable and high-available way. 18 | - [x] Update the spec of the deployed Zookeeper cluster to do adjustments like replicas (scalability) and resources. 19 | - ScaleUp 20 | - ScaleDown 21 | - Rollout 22 | - Observe 23 | - [x] Create Prometheus target for the Zookeeper node. 24 | - [x] Delete the Zookeeper cluster and all the related resources owned by the instance. 25 | 26 | ## Design 27 | 28 | Diagram below shows the overall design of this operator, 29 | 30 | ![架构图](https://raw.githubusercontent.com/Ghostbaby/picgo/master/image2019-11-21_15-38-17.png) 31 | 32 | 33 | For more design details, check the [architecture](design/zookeeper-operator-cn.md) document. 34 | 35 | ## Installation 36 | 37 | You can follow the [installation guide](docs/installation.md) to deploy this operator to your K8s clusters. 38 | 39 | Additionally, follow [sample deployment guide](./docs/sample_deploy_guide.md) to have a try of deploying the sample to your K8s clusters. 40 | 41 | ## Versioning & Dependencies 42 | 43 | | Component \ Versions | 0.5.0 | 1.0.0 | 1.1.0 | 44 | |----------------------|--------|-------|-------| 45 | | **Zookeeper** | 3.5.6 | [TBD] | [TBD] | 46 | | | | | 47 | | agent | 0.0.1 | [TBD] | [TBD] | 48 | 49 | ## Compatibilities 50 | 51 | | Kubernetes / Versions | 0.5.0 | 1.0.0 | 1.1.0 | 52 | |-----------------------|---------|---------|------| 53 | | 1.17 | + | [TBD] | [TBD] | 54 | | 1.18 | + | [TBD] | [TBD] | 55 | | 1.19 | + | [TBD] | [TBD] | 56 | 57 | **Notes:** `+`= verified `-`= not verified 58 | 59 | ## Development 60 | 61 | Interested in contributions? Follow the [CONTRIBUTING](./docs/DEVELOPMENT.md) guide to start on this project. Your contributions will be highly appreciated and creditable. 62 | 63 | ## Community 64 | 65 | * Send mail to Ghostbaby mail: zhuhuijunzhj@gmail.com 66 | 67 | ## Documents 68 | 69 | See documents [here](./docs). 70 | 71 | ## Additional Documents 72 | 73 | * [Kubernetes Operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) 74 | * [Kubebuilder](https://book.kubebuilder.io/) 75 | * [Custom Resource Definition](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) 76 | 77 | 78 | ## License 79 | 80 | [Apache-2.0](https://github.com/Ghostbaby/zookeeper-operator/blob/master/LICENSE) 81 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 24 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/prometheus" 25 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 26 | 27 | "k8s.io/apimachinery/pkg/runtime" 28 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 29 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 32 | 33 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 34 | "github.com/ghostbaby/zookeeper-operator/controllers" 35 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/observer" 36 | // +kubebuilder:scaffold:imports 37 | ) 38 | 39 | var ( 40 | scheme = runtime.NewScheme() 41 | setupLog = ctrl.Log.WithName("setup") 42 | ) 43 | 44 | func init() { 45 | _ = clientgoscheme.AddToScheme(scheme) 46 | 47 | _ = cachev1alpha1.AddToScheme(scheme) 48 | // +kubebuilder:scaffold:scheme 49 | } 50 | 51 | func main() { 52 | var metricsAddr string 53 | var enableLeaderElection bool 54 | flag.StringVar(&metricsAddr, "metrics-addr", ":8081", "The address the metric endpoint binds to.") 55 | flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, 56 | "Enable leader election for controller manager. "+ 57 | "Enabling this will ensure there is only one active controller manager.") 58 | flag.Parse() 59 | 60 | ctrl.SetLogger(zap.New(zap.UseDevMode(true))) 61 | 62 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 63 | Scheme: scheme, 64 | MetricsBindAddress: metricsAddr, 65 | Port: 9443, 66 | LeaderElection: enableLeaderElection, 67 | LeaderElectionID: "4884869b.ghostbaby.io", 68 | }) 69 | if err != nil { 70 | setupLog.Error(err, "unable to start manager") 71 | os.Exit(1) 72 | } 73 | 74 | mcli, err := prometheus.NewRegistry(mgr) 75 | if err != nil { 76 | panic(err) 77 | } 78 | 79 | if err = (&controllers.WorkloadReconciler{ 80 | Client: mgr.GetClient(), 81 | ServiceGetter: &controllers.ServiceGetterImpl{}, 82 | Log: ctrl.Log.WithName("controllers").WithName("Workload"), 83 | Recorder: mgr.GetEventRecorderFor("Workload"), 84 | Scheme: mgr.GetScheme(), 85 | Observers: observer.NewManager(observer.DefaultSettings), 86 | ObservedState: &observer.State{}, 87 | Monitor: mcli, 88 | ZKClient: &zk.BaseClient{}, 89 | Finalizers: finalizer.NewHandler(mgr.GetClient()), 90 | }).SetupWithManager(mgr); err != nil { 91 | setupLog.Error(err, "unable to create controller", "controller", "Workload") 92 | os.Exit(1) 93 | } 94 | if err = (&cachev1alpha1.Workload{}).SetupWebhookWithManager(mgr); err != nil { 95 | setupLog.Error(err, "unable to create webhook", "webhook", "Workload") 96 | os.Exit(1) 97 | } 98 | // +kubebuilder:scaffold:builder 99 | 100 | setupLog.Info("starting manager") 101 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 102 | setupLog.Error(err, "problem running manager") 103 | os.Exit(1) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /controllers/workload/common/finalizer/finalizer.go: -------------------------------------------------------------------------------- 1 | package finalizer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 7 | "k8s.io/apimachinery/pkg/api/meta" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | ctrl "sigs.k8s.io/controller-runtime" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | ) 13 | 14 | var log = ctrl.Log.WithName("controllers").WithName("finalizer") 15 | 16 | // Finalizer can be attached to a resource and executed upon resource deletion 17 | type Finalizer struct { 18 | Name string 19 | Execute func() error 20 | } 21 | 22 | // Handler handles registration and execution of finalizers 23 | // Note that it is not thread-safe. 24 | type Handler struct { 25 | client client.Client 26 | } 27 | 28 | // NewHandler creates a Handler 29 | func NewHandler(client client.Client) Handler { 30 | return Handler{ 31 | client: client, 32 | } 33 | } 34 | 35 | // Handle the configured finalizers for the given resource. 36 | // The given objectMeta must be a sub-part of the given resource: updates for the 37 | // resource will be issued against the apiserver based on objectMeta content if needed. 38 | // If the resource is marked for deletion, finalizers will be executed then removed 39 | // from the resource. 40 | // Else, the function is making sure all finalizers are correctly registered for the resource. 41 | func (h *Handler) Handle(resource runtime.Object, finalizers ...Finalizer) error { 42 | metaObject, err := meta.Accessor(resource) 43 | if err != nil { 44 | return err 45 | } 46 | var needUpdate bool 47 | var finalizerErr error 48 | if metaObject.GetDeletionTimestamp().IsZero() { 49 | // resource is not being deleted, make sure all finalizers are there 50 | needUpdate = h.reconcileFinalizers(finalizers, metaObject) 51 | } else { 52 | // resource is being deleted, let's execute finalizers 53 | needUpdate, finalizerErr = h.executeFinalizers(finalizers, metaObject) 54 | } 55 | if needUpdate { 56 | if updateErr := h.client.Update(context.TODO(), resource); updateErr != nil { 57 | return updateErr 58 | } 59 | } 60 | return finalizerErr 61 | } 62 | 63 | // reconcileFinalizers ensures all finalizers exist in the given objectMeta. 64 | // Returns a bool indicating if an update is required to the object 65 | func (h *Handler) reconcileFinalizers(finalizers []Finalizer, object metav1.Object) bool { 66 | needUpdate := false 67 | for _, finalizer := range finalizers { 68 | // add finalizer if not already there 69 | if !utils.StringInSlice(finalizer.Name, object.GetFinalizers()) { 70 | log.Info("Registering finalizer", "finalizer_name", finalizer.Name, "namespace", object.GetNamespace(), "name", object.GetName()) 71 | object.SetFinalizers(append(object.GetFinalizers(), finalizer.Name)) 72 | needUpdate = true 73 | } 74 | } 75 | return needUpdate 76 | } 77 | 78 | // executeFinalizers runs all registered finalizers in the given objectMeta. 79 | // Once a finalizer is executed, it is removed from the objectMeta's list, 80 | // and an update to the apiserver is issued for the given resource. 81 | func (h *Handler) executeFinalizers(finalizers []Finalizer, object metav1.Object) (bool, error) { 82 | needUpdate := false 83 | var finalizerErr error 84 | for _, finalizer := range finalizers { 85 | // for each registered finalizer, execute it, then remove from the list 86 | if utils.StringInSlice(finalizer.Name, object.GetFinalizers()) { 87 | log.Info("Executing finalizer", "finalizer_name", finalizer.Name, "namespace", object.GetNamespace(), "name", object.GetName()) 88 | if finalizerErr = finalizer.Execute(); finalizerErr != nil { 89 | break 90 | } 91 | needUpdate = true 92 | object.SetFinalizers(utils.RemoveStringInSlice(finalizer.Name, object.GetFinalizers())) 93 | } 94 | } 95 | return needUpdate, finalizerErr 96 | } 97 | -------------------------------------------------------------------------------- /controllers/workload/getter.go: -------------------------------------------------------------------------------- 1 | package workload 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/prometheus" 7 | 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/rollout" 9 | 10 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/scale" 11 | 12 | appsv1 "k8s.io/api/apps/v1" 13 | 14 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/finalizer" 15 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 16 | 17 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/observer" 18 | 19 | cachev1alpha1 "github.com/ghostbaby/zookeeper-operator/api/v1alpha1" 20 | "github.com/ghostbaby/zookeeper-operator/controllers/k8s" 21 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/provision" 22 | "github.com/go-logr/logr" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | "k8s.io/client-go/tools/record" 25 | ) 26 | 27 | type Reconciler interface { 28 | // Reconcile the dependent service. 29 | Reconcile() error 30 | } 31 | 32 | type Getter interface { 33 | // For Provision 34 | ProvisionWorkload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler 35 | 36 | // For Scale 37 | ScaleWorkload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler 38 | 39 | // For Rollout 40 | RolloutWorkload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler 41 | } 42 | 43 | type GetOptions struct { 44 | Client k8s.Client 45 | Recorder record.EventRecorder 46 | Log logr.Logger 47 | DClient k8s.DClient 48 | Scheme *runtime.Scheme 49 | Labels map[string]string 50 | Observers *observer.Manager 51 | ZKClient *zk.BaseClient 52 | ObservedState *observer.State 53 | Monitor *prometheus.GenericClientset 54 | Finalizers finalizer.Handler 55 | ExpectSts *appsv1.StatefulSet 56 | ActualSts *appsv1.StatefulSet 57 | } 58 | 59 | type GetterImpl struct { 60 | } 61 | 62 | func (w *ReconcileWorkload) GetOptions() *GetOptions { 63 | return &GetOptions{ 64 | Client: w.Client, 65 | Recorder: w.Recorder, 66 | Log: w.Log, 67 | DClient: w.DClient, 68 | Scheme: w.Scheme, 69 | Labels: w.Labels, 70 | Observers: w.Observers, 71 | ZKClient: w.ZKClient, 72 | ObservedState: w.ObservedState, 73 | Finalizers: w.Finalizers, 74 | Monitor: w.Monitor, 75 | } 76 | } 77 | 78 | func (impl *GetterImpl) ProvisionWorkload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler { 79 | return &provision.Provision{ 80 | Workload: workload, 81 | CTX: ctx, 82 | Client: options.Client, 83 | Recorder: options.Recorder, 84 | Log: options.Log, 85 | Labels: options.Labels, 86 | Scheme: options.Scheme, 87 | Observers: options.Observers, 88 | ZKClient: options.ZKClient, 89 | ObservedState: options.ObservedState, 90 | Finalizers: options.Finalizers, 91 | Monitor: options.Monitor, 92 | } 93 | } 94 | 95 | func (impl *GetterImpl) ScaleWorkload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler { 96 | return &scale.Scale{ 97 | Workload: workload, 98 | Client: options.Client, 99 | Recorder: options.Recorder, 100 | Log: options.Log, 101 | Labels: options.Labels, 102 | Scheme: options.Scheme, 103 | } 104 | } 105 | 106 | func (impl *GetterImpl) RolloutWorkload(ctx context.Context, workload *cachev1alpha1.Workload, options *GetOptions) Reconciler { 107 | return &rollout.Rollout{ 108 | Workload: workload, 109 | Client: options.Client, 110 | Recorder: options.Recorder, 111 | Log: options.Log, 112 | Labels: options.Labels, 113 | Scheme: options.Scheme, 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /controllers/k8s/client_test.go: -------------------------------------------------------------------------------- 1 | package k8s_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/ghostbaby/zookeeper-operator/controllers/k8s" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/apimachinery/pkg/types" 13 | "sigs.k8s.io/controller-runtime/pkg/client" 14 | ) 15 | 16 | type ctxKey struct{} 17 | 18 | var ( 19 | userProvidedContextKey = ctxKey{} 20 | errUsingUserProvidedContext = errors.New("using user-provided context") 21 | errUsingDefaultTimeoutContext = errors.New("using default timeout context") 22 | ) 23 | 24 | var ( 25 | tests []struct { 26 | name string 27 | call func(c k8s.Client) error 28 | } 29 | ctx = context.Background() 30 | ) 31 | 32 | var _ = Describe("Client", func() { 33 | BeforeEach(func() { 34 | tests = []struct { 35 | name string 36 | call func(c k8s.Client) error 37 | }{ 38 | { 39 | name: "Get", 40 | call: func(c k8s.Client) error { 41 | return c.Get(types.NamespacedName{}, nil) 42 | }, 43 | }, 44 | { 45 | name: "List", 46 | call: func(c k8s.Client) error { 47 | return c.List(nil, nil) 48 | }, 49 | }, 50 | { 51 | name: "Create", 52 | call: func(c k8s.Client) error { 53 | return c.Create(nil) 54 | }, 55 | }, 56 | { 57 | name: "Update", 58 | call: func(c k8s.Client) error { 59 | return c.Update(nil) 60 | }, 61 | }, 62 | { 63 | name: "Patch", 64 | call: func(c k8s.Client) error { 65 | return c.Patch(nil, nil, nil) 66 | }, 67 | }, 68 | } 69 | }) 70 | 71 | Describe("Wrapper k8s Client", func() { 72 | Context("Wrapper k8s Client", func() { 73 | It("should pass ", func() { 74 | for _, tt := range tests { 75 | // setup the Client with a timeout 76 | c := k8s.WrapClient(ctx, mockedClient{}) 77 | 78 | // pass a custom context with the call 79 | ctx := context.WithValue(context.Background(), userProvidedContextKey, userProvidedContextKey) 80 | err := tt.call(c.WithContext(ctx)) 81 | // make sure this custom context was used and not the timeout one 82 | Expect(err).To(Equal(errUsingUserProvidedContext)) 83 | } 84 | }) 85 | }) 86 | 87 | }) 88 | }) 89 | 90 | // mockedClient's only purpose is to perform checks against the context 91 | // passed in from the surrounding Client 92 | type mockedClient struct{} 93 | 94 | func (m mockedClient) checkCtx(ctx context.Context) error { 95 | if ctx == nil { 96 | return errors.New("using no context") 97 | } 98 | if ctx.Value(userProvidedContextKey) == userProvidedContextKey { 99 | return errUsingUserProvidedContext 100 | } 101 | // should be the init timeout context 102 | <-ctx.Done() 103 | return errUsingDefaultTimeoutContext 104 | } 105 | 106 | func (m mockedClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { 107 | return m.checkCtx(ctx) 108 | } 109 | 110 | func (m mockedClient) List(ctx context.Context, list runtime.Object, opts ...client.ListOption) error { 111 | return m.checkCtx(ctx) 112 | } 113 | 114 | func (m mockedClient) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error { 115 | return m.checkCtx(ctx) 116 | } 117 | 118 | func (m mockedClient) Delete(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error { 119 | return m.checkCtx(ctx) 120 | } 121 | 122 | func (m mockedClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { 123 | return m.checkCtx(ctx) 124 | } 125 | 126 | func (m mockedClient) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error { 127 | return m.checkCtx(ctx) 128 | } 129 | 130 | func (m mockedClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...client.DeleteAllOfOption) error { 131 | return m.checkCtx(ctx) 132 | } 133 | 134 | func (m mockedClient) Status() client.StatusWriter { 135 | return mockedStatusWriter{c: m} 136 | } 137 | 138 | type mockedStatusWriter struct { 139 | c mockedClient 140 | } 141 | 142 | func (m mockedStatusWriter) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { 143 | return m.c.checkCtx(ctx) 144 | } 145 | 146 | func (m mockedStatusWriter) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error { 147 | return m.c.checkCtx(ctx) 148 | } 149 | -------------------------------------------------------------------------------- /controllers/workload/scale/reconfig.go: -------------------------------------------------------------------------------- 1 | package scale 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "math/rand" 8 | "net/http" 9 | "strings" 10 | 11 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 12 | 13 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/observer" 14 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 15 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 16 | zkcli "github.com/samuel/go-zookeeper/zk" 17 | "gopkg.in/fatih/set.v0" 18 | ) 19 | 20 | type AddMember struct { 21 | Record string `json:"record"` 22 | } 23 | 24 | func (s *Scale) ReConfig() error { 25 | name := s.Workload.GetName() 26 | namespace := s.Workload.GetNamespace() 27 | 28 | //获取目前正在允许的pod信息 29 | currentPods, err := utils.GetCurrentPods(s.Client, s.Workload, s.Labels, s.Log) 30 | if err != nil { 31 | s.Log.Info( 32 | "Unable to get current pods.", 33 | "error", err, 34 | "namespace", namespace, 35 | "zk_name", name, 36 | ) 37 | return errors.New("reconfig need requeue") 38 | } 39 | 40 | //获取期望 zk config内容 41 | podNames := utils.PodNames(*s.ExpectSts) 42 | AddMemberRecords := utils.GetPodIp(s.Workload, podNames) 43 | expectConfigRecord := set.New(set.ThreadSafe) 44 | 45 | for _, v := range AddMemberRecords { 46 | expectConfigRecord.Add(v) 47 | } 48 | 49 | if len(currentPods) == 0 { 50 | s.Log.Info("Cluster is building ,pls hold on.") 51 | return nil 52 | } 53 | //生成zk connect client 54 | randomPod := currentPods[rand.Intn(len(currentPods))] 55 | podIp := randomPod.Status.PodIP 56 | 57 | zkAgentUrl := fmt.Sprintf("http://%s:%d", podIp, model.AgentPort) 58 | 59 | cli := &zk.BaseClient{ 60 | HTTP: &http.Client{}, 61 | Endpoint: zkAgentUrl, 62 | Transport: &http.Transport{}, 63 | } 64 | 65 | //获取实际 zk config内容 66 | timeoutCtx, cancel := context.WithTimeout(context.Background(), observer.DefaultSettings.RequestTimeout) 67 | defer cancel() 68 | var config AddMember 69 | if err := cli.Get(timeoutCtx, "/get", &config); err != nil { 70 | s.Log.Info( 71 | "Unable to get zk config.", 72 | "error", err, 73 | "url", zkAgentUrl, 74 | "pod", randomPod.Name, 75 | ) 76 | return err 77 | } 78 | 79 | //返回config字符串转数组 80 | currentConfigs := strings.Split(config.Record, "\n") 81 | 82 | //去掉config 末尾 version=100000104 项 83 | currentConfigs = currentConfigs[:len(currentConfigs)-1] 84 | 85 | //config数组转set interface{} 86 | currentConfigRecord := set.New(set.ThreadSafe) 87 | for _, v := range currentConfigs { 88 | currentConfigRecord.Add(v) 89 | } 90 | 91 | //比较实际和期待 zk config 92 | //needAdd为需要添加配置项 93 | //needDel为需要删除配置项 94 | needAdd := set.Difference(expectConfigRecord, currentConfigRecord) 95 | needDel := set.Difference(currentConfigRecord, expectConfigRecord) 96 | 97 | //set interface转[]string 98 | needAddArray := set.StringSlice(needAdd) 99 | needDelArray := set.StringSlice(needDel) 100 | 101 | //如果待添加和待删除数组均为0，退出reconfig流程 102 | if len(needAddArray) == 0 && len(needDelArray) == 0 { 103 | s.Log.Info( 104 | "Don't need to update zk config.", 105 | "url", zkAgentUrl, 106 | "pod", randomPod.Name, 107 | ) 108 | return nil 109 | } 110 | 111 | //调用zk-agent接口添加新节点到集群中reconfig 112 | for _, record := range needAddArray { 113 | member := &AddMember{ 114 | Record: record, 115 | } 116 | timeoutCtx, cancel := context.WithTimeout(context.Background(), observer.DefaultSettings.RequestTimeout) 117 | defer cancel() 118 | 119 | var result zkcli.Stat 120 | 121 | if err := cli.Post(timeoutCtx, "/add", member, &result); err != nil { 122 | s.Log.Info( 123 | "Unable to add member to zk.", 124 | "error", err, 125 | "url", zkAgentUrl, 126 | "pod", randomPod.Name, 127 | ) 128 | continue 129 | } 130 | } 131 | 132 | //调用zk-agent接口删除新节点到集群中reconfig 133 | for _, record := range needDelArray { 134 | member := &AddMember{ 135 | Record: record, 136 | } 137 | timeoutCtx, cancel := context.WithTimeout(context.Background(), observer.DefaultSettings.RequestTimeout) 138 | defer cancel() 139 | 140 | var result zkcli.Stat 141 | 142 | if err := cli.Post(timeoutCtx, "/del", member, &result); err != nil { 143 | s.Log.Info( 144 | "Unable to add member to zk.", 145 | "error", err, 146 | "url", zkAgentUrl, 147 | "pod", randomPod.Name, 148 | ) 149 | continue 150 | } 151 | } 152 | 153 | return nil 154 | } 155 | -------------------------------------------------------------------------------- /controllers/workload/provision/observer.go: -------------------------------------------------------------------------------- 1 | package provision 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | 9 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 10 | 11 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/zk" 12 | 13 | "time" 14 | 15 | "github.com/pkg/errors" 16 | corev1 "k8s.io/api/core/v1" 17 | "k8s.io/apimachinery/pkg/types" 18 | "k8s.io/client-go/util/workqueue" 19 | "sigs.k8s.io/controller-runtime/pkg/event" 20 | "sigs.k8s.io/controller-runtime/pkg/handler" 21 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 22 | ) 23 | 24 | const ( 25 | DefaultObservationInterval = 10 * time.Second 26 | DefaultRequestTimeout = 1 * time.Minute 27 | ) 28 | 29 | // ResourcesState contains information about a deployments resources. 30 | type ResourcesState struct { 31 | // AllPods are all the pods related to the cluster, including ones with a 32 | // DeletionTimestamp tombstone set. 33 | AllPods []corev1.Pod 34 | // CurrentPods are all non-deleted pods. 35 | CurrentPods []corev1.Pod 36 | // CurrentPodsByPhase are all non-deleted indexed by their PodPhase 37 | CurrentPodsByPhase map[corev1.PodPhase][]corev1.Pod 38 | // DeletingPods are all deleted pods. 39 | DeletingPods []corev1.Pod 40 | } 41 | 42 | func (p *Provision) Observer() error { 43 | var zkUrls []string 44 | var zkUrl string 45 | 46 | _, podList, err := utils.GetStatefulSetPods(p.Client, p.Workload, p.Labels, p.Log) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | if len(podList.Items) == 0 { 52 | p.Log.Info("pod list is empty，pls wait.") 53 | return nil 54 | } 55 | 56 | podArray := podList.Items 57 | 58 | //deletingPods := make([]corev1.Pod, 0) 59 | currentPods := make([]corev1.Pod, 0, len(podArray)) 60 | currentPodsByPhase := make(map[corev1.PodPhase][]corev1.Pod) 61 | 62 | for _, p := range podArray { 63 | //if p.DeletionTimestamp != nil { 64 | // deletingPods = append(deletingPods, p) 65 | // continue 66 | //} 67 | currentPods = append(currentPods, p) 68 | podsInPhase, ok := currentPodsByPhase[p.Status.Phase] 69 | if !ok { 70 | podsInPhase = []corev1.Pod{p} 71 | } else { 72 | podsInPhase = append(podsInPhase, p) 73 | } 74 | currentPodsByPhase[p.Status.Phase] = podsInPhase 75 | } 76 | 77 | //podState := ResourcesState{ 78 | // AllPods: podArray, 79 | // CurrentPods: currentPods, 80 | // CurrentPodsByPhase: currentPodsByPhase, 81 | // DeletingPods: deletingPods, 82 | //} 83 | 84 | if len(currentPods) == 0 { 85 | return fmt.Errorf("current pods is zero, pls waitting....") 86 | } 87 | 88 | zkUrl, zkUrls = utils.GetServiceUrl(p.Workload, currentPods) 89 | 90 | cli := &zk.BaseClient{ 91 | Endpoints: zkUrls, 92 | HTTP: &http.Client{}, 93 | Endpoint: zkUrl, 94 | Transport: &http.Transport{}, 95 | } 96 | 97 | state := p.Observers.ObservedStateResolver( 98 | utils.ExtractNamespacedName(p.Workload), 99 | *cli, 100 | ) 101 | 102 | if state.ClusterStats != nil { 103 | p.Workload.Status.LeaderNode = state.ClusterStats.LeaderNode 104 | p.Workload.Status.AvailableNodes = state.ClusterStats.AvailableNodes 105 | } 106 | 107 | p.Workload.Status.LastTransitionTime = metav1.Now() 108 | 109 | p.ObservedState = &state 110 | p.ZKClient = cli 111 | 112 | if p.Workload.Status.ObservedGeneration != p.Workload.Generation { 113 | p.Workload.Status.ObservedGeneration = p.Workload.Generation 114 | } 115 | 116 | return p.writeStatus() 117 | } 118 | 119 | func (p *Provision) writeStatus() error { 120 | err := p.Client.WriteStatus(p.Workload) 121 | if err != nil { 122 | // may be it's k8s v1.10 and erlier (e.g. oc3.9) that doesn't support status updates 123 | // so try to update whole CR 124 | err := p.Client.Update(p.Workload) 125 | if err != nil { 126 | return errors.Wrap(err, "send update") 127 | } 128 | } 129 | 130 | return nil 131 | } 132 | 133 | // GenericEventHandler returns an EventHandler that enqueues a reconciliation request 134 | // from the generic event NamespacedName. 135 | func GenericEventHandler() handler.EventHandler { 136 | return handler.Funcs{ 137 | GenericFunc: func(evt event.GenericEvent, q workqueue.RateLimitingInterface) { 138 | q.Add(reconcile.Request{ 139 | NamespacedName: types.NamespacedName{ 140 | Namespace: evt.Meta.GetNamespace(), 141 | Name: evt.Meta.GetName(), 142 | }, 143 | }) 144 | }, 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /controllers/workload/common/sts/new.go: -------------------------------------------------------------------------------- 1 | package sts 2 | 3 | import ( 4 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/cm" 5 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 6 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/volume" 7 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/model" 8 | appsv1 "k8s.io/api/apps/v1" 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | ) 12 | 13 | func (s *STS) GenerateStatefulset() (*appsv1.StatefulSet, error) { 14 | name := s.Workload.GetName() 15 | namespace := s.Workload.GetNamespace() 16 | 17 | sts := NewStatefulSet(name, namespace) 18 | resource := getResources(s.Workload) 19 | 20 | var volumes []corev1.Volume 21 | var volumeMount []corev1.VolumeMount 22 | var image string 23 | var initContainerArray []corev1.Container 24 | 25 | if s.Workload.Spec.Image != "" { 26 | image = s.Workload.Spec.Image 27 | } else { 28 | image = utils.Joins(model.DefaultImageRepository, ":", s.Workload.Spec.Version) 29 | } 30 | 31 | //生成agent配置文件 32 | agentConfigFileVolume := volume.ConfigMapVolume{ 33 | CmName: genConfigMapName(name, cm.AgentVolumeName), 34 | Name: cm.AgentVolumeName, 35 | MountPath: cm.AgentVolumeMountPath, 36 | DefaultMode: 0755, 37 | } 38 | volumes = append(volumes, agentConfigFileVolume.Volume()) 39 | volumeMount = append(volumeMount, agentConfigFileVolume.VolumeMount()) 40 | 41 | //生成data存储卷 42 | dataVolume := GetDataVolume(s.Workload) 43 | if dataVolume != nil { 44 | volumes = append(volumes, *dataVolume) 45 | } 46 | containerDefaultVM := BuildDefaultVolumes() 47 | 48 | volumeMount = append(volumeMount, containerDefaultVM...) 49 | volumes = append(volumes, cm.PluginVolumes.Volumes()...) 50 | volumes = append(volumes, volume.DefaultLogsVolume) 51 | 52 | //生成初始化脚本存储卷 53 | scriptsVolume := volume.ConfigMapVolume{ 54 | CmName: genConfigMapName(name, cm.ScriptsVolumeName), 55 | Name: cm.ScriptsVolumeName, 56 | MountPath: cm.ScriptsVolumeMountPath, 57 | DefaultMode: 0755, 58 | } 59 | 60 | volumes = append(volumes, scriptsVolume.Volume()) 61 | volumeMount = append(volumeMount, scriptsVolume.VolumeMount()) 62 | 63 | //生成动态配置存储卷 64 | dynamicConfigFileVolume := volume.ConfigMapVolume{ 65 | CmName: genConfigMapName(name, cm.DynamicConfigFileVolumeName), 66 | Name: cm.DynamicConfigFileVolumeName, 67 | MountPath: cm.DynamicConfigFileVolumeMountPath, 68 | DefaultMode: 0755, 69 | } 70 | volumes = append(volumes, dynamicConfigFileVolume.Volume()) 71 | volumeMount = append(volumeMount, dynamicConfigFileVolume.VolumeMount()) 72 | 73 | //生成启动配置文件 74 | ConfigVolume := volume.ConfigMapVolume{ 75 | CmName: genConfigMapName(name, cm.ConfigVolumeName), 76 | Name: cm.ConfigVolumeName, 77 | MountPath: cm.ConfigVolumeMountPath, 78 | DefaultMode: 0755, 79 | } 80 | 81 | volumes = append(volumes, ConfigVolume.Volume()) 82 | volumeMount = append(volumeMount, ConfigVolume.VolumeMount()) 83 | 84 | //生成初始化容器 85 | prepareFsContainer, err := NewPrepareFSInitContainer(image) 86 | if err != nil { 87 | return nil, err 88 | } 89 | initContainerArray = append(initContainerArray, prepareFsContainer) 90 | 91 | sts.Spec = appsv1.StatefulSetSpec{ 92 | ServiceName: name, 93 | Replicas: s.Workload.Spec.Replicas, 94 | Selector: &metav1.LabelSelector{ 95 | MatchLabels: s.Labels, 96 | }, 97 | Template: corev1.PodTemplateSpec{ 98 | ObjectMeta: metav1.ObjectMeta{ 99 | Labels: s.Labels, 100 | Annotations: s.Workload.Spec.Annotations, 101 | }, 102 | Spec: corev1.PodSpec{ 103 | Affinity: PodAffinity(s.Workload.Spec.Affinity, s.Labels), 104 | NodeSelector: s.Workload.Spec.NodeSelector, 105 | Tolerations: s.Workload.Spec.Tolerations, 106 | PriorityClassName: s.Workload.Spec.PriorityClassName, 107 | RestartPolicy: corev1.RestartPolicyAlways, 108 | InitContainers: initContainerArray, 109 | Containers: []corev1.Container{ 110 | container(resource, image, volumeMount), 111 | }, 112 | Volumes: volumes, 113 | }, 114 | }, 115 | UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ 116 | Type: appsv1.OnDeleteStatefulSetStrategyType, 117 | }, 118 | PodManagementPolicy: appsv1.ParallelPodManagement, 119 | } 120 | 121 | // todo：check is spec.cluster.exporter defined 122 | if s.Workload.Spec.Cluster.Exporter.Exporter { 123 | exporter := s.CreateExporterContainer(s.Workload) 124 | sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, exporter) 125 | } 126 | 127 | //生成agent容器 128 | agentContainer := s.CreateAgentContainer(agentConfigFileVolume.VolumeMount()) 129 | sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, agentContainer) 130 | 131 | if s.Workload.Spec.Cluster.Storage.PersistentVolumeClaim != nil { 132 | sts.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ 133 | *s.Workload.Spec.Cluster.Storage.PersistentVolumeClaim, 134 | } 135 | } 136 | 137 | return sts, nil 138 | } 139 | -------------------------------------------------------------------------------- /controllers/workload/common/cm/init_script.go: -------------------------------------------------------------------------------- 1 | package cm 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | 7 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/utils" 8 | "github.com/ghostbaby/zookeeper-operator/controllers/workload/common/volume" 9 | corev1 "k8s.io/api/core/v1" 10 | ) 11 | 12 | var ( 13 | ConfigVolume = SharedVolume{ 14 | Name: "zookeeper-internal-config-local", 15 | InitContainerMountPath: "/mnt/zookeeper/zookeeper-config-local", 16 | ContainerMountPath: volume.ConfigVolumeMountPath, 17 | } 18 | 19 | PluginVolumes = SharedVolumeArray{ 20 | Array: []SharedVolume{ 21 | ConfigVolume, 22 | }, 23 | } 24 | 25 | // linkedFiles describe how various secrets are mapped into the pod's filesystem. 26 | linkedFiles = LinkedFilesArray{ 27 | Array: []LinkedFile{ 28 | { 29 | Source: utils.Joins(ConfigVolumeMountPath, "/", ConfigFileName), 30 | Target: utils.Joins(ConfigVolume.ContainerMountPath, "/", ConfigFileName), 31 | }, 32 | { 33 | Source: utils.Joins(DynamicConfigFileVolumeMountPath, "/", DynamicConfigFile), 34 | Target: utils.Joins(ConfigVolume.ContainerMountPath, "/", DynamicConfigFile), 35 | }, 36 | }, 37 | } 38 | ) 39 | 40 | func (v SharedVolumeArray) Volumes() []corev1.Volume { 41 | volumes := make([]corev1.Volume, len(v.Array)) 42 | for i, v := range v.Array { 43 | volumes[i] = corev1.Volume{ 44 | Name: v.Name, 45 | VolumeSource: corev1.VolumeSource{ 46 | EmptyDir: &corev1.EmptyDirVolumeSource{}, 47 | }, 48 | } 49 | } 50 | return volumes 51 | } 52 | 53 | // SharedVolumes represents a list of SharedVolume 54 | type SharedVolumeArray struct { 55 | Array []SharedVolume 56 | } 57 | 58 | // SharedVolume between the init container and the ZK container. 59 | type SharedVolume struct { 60 | Name string // Volume name 61 | InitContainerMountPath string // Mount path in the init container 62 | ContainerMountPath string // Mount path in the zookeeper container 63 | } 64 | 65 | // LinkedFilesArray contains all files to be linked in the init container. 66 | type LinkedFilesArray struct { 67 | Array []LinkedFile 68 | } 69 | 70 | // LinkedFile describes a symbolic link with source and target. 71 | type LinkedFile struct { 72 | Source string 73 | Target string 74 | } 75 | 76 | // TemplateParams are the parameters manipulated in the scriptTemplate 77 | type TemplateParams struct { 78 | // SharedVolumes are directories to persist in shared volumes 79 | PluginVolumes SharedVolumeArray 80 | // LinkedFiles are files to link individually 81 | LinkedFiles LinkedFilesArray 82 | // ChownToElasticsearch are paths that need to be chowned to the Elasticsearch user/group. 83 | ChownToZookeeper []string 84 | } 85 | 86 | // RenderScriptTemplate renders scriptTemplate using the given TemplateParams 87 | func RenderScriptTemplate(params TemplateParams) (string, error) { 88 | tplBuffer := bytes.Buffer{} 89 | 90 | if err := scriptTemplate.Execute(&tplBuffer, params); err != nil { 91 | return "", err 92 | } 93 | return tplBuffer.String(), nil 94 | } 95 | 96 | const ( 97 | PrepareFsScriptConfigKey = "prepare-fs.sh" 98 | ) 99 | 100 | // scriptTemplate is the main script to be run 101 | // in the prepare-fs init container before ES starts 102 | var scriptTemplate = template.Must(template.New("").Parse( 103 | `#!/usr/bin/env bash 104 | 105 | set -eu 106 | 107 | # compute time in seconds since the given start time 108 | function duration() { 109 | local start=$1 110 | end=$(date +%s) 111 | echo $((end-start)) 112 | } 113 | 114 | ###################### 115 | # START # 116 | ###################### 117 | 118 | script_start=$(date +%s) 119 | 120 | echo "Starting init script" 121 | 122 | ###################### 123 | # Config linking # 124 | ###################### 125 | 126 | # Link individual files from their mount location into the config dir 127 | # to a volume, to be used by the ZK container 128 | ln_start=$(date +%s) 129 | {{range .LinkedFiles.Array}} 130 | echo "Linking {{.Source}} to {{.Target}}" 131 | ln -sf {{.Source}} {{.Target}} 132 | {{end}} 133 | ls -l /conf 134 | echo "File linking duration: $(duration $ln_start) sec." 135 | 136 | ###################### 137 | # Volumes chown # 138 | ###################### 139 | 140 | # chown the data and logs volume to the elasticsearch user 141 | # only done when running as root, other cases should be handled 142 | # with a proper security context 143 | chown_start=$(date +%s) 144 | if [[ $EUID -eq 0 ]]; then 145 | {{range .ChownToZookeeper}} 146 | echo "chowning {{.}} to zookeeper:zookeeper" 147 | chown -v zookeeper:zookeeper {{.}} 148 | {{end}} 149 | fi 150 | echo "chown duration: $(duration $chown_start) sec." 151 | 152 | ###################### 153 | # End # 154 | ###################### 155 | 156 | echo "Init script successful" 157 | echo "Script duration: $(duration $script_start) sec." 158 | `)) 159 | 160 | func RenderPrepareFsScript() (string, error) { 161 | return RenderScriptTemplate(TemplateParams{ 162 | PluginVolumes: PluginVolumes, 163 | LinkedFiles: linkedFiles, 164 | ChownToZookeeper: []string{ 165 | DataMountPath, 166 | LogsMountPath, 167 | }, 168 | }) 169 | } 170 | -------------------------------------------------------------------------------- /agent/http/run.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "regexp" 11 | "time" 12 | 13 | "github.com/ghostbaby/zk-agent/api" 14 | 15 | "github.com/ghostbaby/zk-agent/g" 16 | 17 | "github.com/samuel/go-zookeeper/zk" 18 | ) 19 | 20 | type AddMember struct { 21 | Record string `json:"record"` 22 | } 23 | 24 | func getZkStatus(w http.ResponseWriter, r *http.Request) { 25 | 26 | if !g.Config().Http.Backdoor { 27 | w.Write([]byte("/run disabled")) 28 | return 29 | } 30 | 31 | c, _ := zk.FLWSrvr([]string{g.Config().ZkHost}, time.Second*10) //*10) 32 | 33 | var out []byte 34 | var err error 35 | for _, v := range c { 36 | out, err = json.Marshal(v) 37 | if err != nil { 38 | w.Write([]byte("exec fail: " + err.Error())) 39 | return 40 | } 41 | } 42 | 43 | w.Write(out) 44 | 45 | } 46 | 47 | func Health(w http.ResponseWriter, r *http.Request) { 48 | 49 | if err := api.WriteJSON(w, "healthy"); err != nil { 50 | log.Printf("Failed to write response: %v", err) 51 | return 52 | } 53 | 54 | } 55 | 56 | func getZkClient(w http.ResponseWriter, r *http.Request) { 57 | if !g.Config().Http.Backdoor { 58 | w.Write([]byte("/run disabled")) 59 | return 60 | } 61 | 62 | c, _ := zk.FLWCons([]string{g.Config().ZkHost}, time.Second*10) //*10) 63 | 64 | var out []byte 65 | var err error 66 | for _, v := range c { 67 | out, err = json.Marshal(v) 68 | if err != nil { 69 | w.Write([]byte("exec fail: " + err.Error())) 70 | return 71 | } 72 | } 73 | 74 | w.Write(out) 75 | 76 | } 77 | 78 | func getZkRunok(w http.ResponseWriter, r *http.Request) { 79 | 80 | if !g.Config().Http.Backdoor { 81 | w.Write([]byte("/run disabled")) 82 | return 83 | } 84 | 85 | c := zk.FLWRuok([]string{g.Config().ZkHost}, time.Second*10) //*10) 86 | 87 | var out []byte 88 | var err error 89 | for _, v := range c { 90 | out, err = json.Marshal(v) 91 | if err != nil { 92 | w.Write([]byte("exec fail: " + err.Error())) 93 | return 94 | } 95 | } 96 | 97 | w.Write(out) 98 | 99 | } 100 | 101 | func addMember(w http.ResponseWriter, r *http.Request) { 102 | if r.ContentLength == 0 { 103 | http.Error(w, "body is blank", http.StatusBadRequest) 104 | return 105 | } 106 | 107 | bs, err := ioutil.ReadAll(r.Body) 108 | 109 | if err != nil { 110 | http.Error(w, err.Error(), http.StatusInternalServerError) 111 | return 112 | } 113 | 114 | record := &AddMember{} 115 | if err := json.Unmarshal(bs, record); err != nil { 116 | w.Write([]byte("exec fail: " + err.Error())) 117 | return 118 | } 119 | 120 | body := record.Record 121 | 122 | zkConnect, _, err := zk.Connect([]string{g.Config().ZkHost}, time.Second*10) 123 | if err != nil { 124 | w.Write([]byte("exec fail: " + err.Error())) 125 | return 126 | } 127 | defer zkConnect.Close() 128 | var reconfigData []string 129 | reconfigData = append(reconfigData, body) 130 | 131 | out, err := zkConnect.IncrementalReconfig(reconfigData, nil, -1) 132 | if err != nil { 133 | w.Write([]byte("exec fail: " + err.Error())) 134 | return 135 | } 136 | request, _ := json.Marshal(out) 137 | 138 | w.Write(request) 139 | } 140 | 141 | func getMember(w http.ResponseWriter, r *http.Request) { 142 | 143 | var out []byte 144 | var err error 145 | 146 | if !g.Config().Http.Backdoor { 147 | w.Write([]byte("/run disabled")) 148 | return 149 | } 150 | 151 | zkConnect, _, err := zk.Connect([]string{g.Config().ZkHost}, time.Second*10) 152 | if err != nil { 153 | w.Write([]byte("exec fail: " + err.Error())) 154 | return 155 | } 156 | defer zkConnect.Close() 157 | 158 | out, _, err = zkConnect.Get("/zookeeper/config") 159 | if err != nil { 160 | w.Write([]byte("exec fail: " + err.Error())) 161 | return 162 | } 163 | 164 | record := &AddMember{ 165 | Record: string(out), 166 | } 167 | 168 | recordByte, err := json.Marshal(record) 169 | if err != nil { 170 | w.Write([]byte("exec fail: " + err.Error())) 171 | return 172 | } 173 | 174 | w.Write(recordByte) 175 | 176 | } 177 | 178 | func delMember(w http.ResponseWriter, r *http.Request) { 179 | 180 | if r.ContentLength == 0 { 181 | http.Error(w, "body is blank", http.StatusBadRequest) 182 | return 183 | } 184 | 185 | bs, err := ioutil.ReadAll(r.Body) 186 | 187 | if err != nil { 188 | http.Error(w, err.Error(), http.StatusInternalServerError) 189 | return 190 | } 191 | 192 | record := &AddMember{} 193 | if err := json.Unmarshal(bs, record); err != nil { 194 | w.Write([]byte("exec fail: " + err.Error())) 195 | return 196 | } 197 | 198 | body := record.Record 199 | 200 | //获取需要删除的myid 201 | zkConfigRegexp := regexp.MustCompile(`^server.(\d+)=.*2181$`) 202 | params := zkConfigRegexp.FindStringSubmatch(body) 203 | for _, param := range params { 204 | fmt.Println(param) 205 | } 206 | 207 | if len(params) <= 0 { 208 | err := errors.New("unable to get zk node id") 209 | w.Write([]byte("exec fail: " + err.Error())) 210 | return 211 | } 212 | 213 | zkNodeID := params[1] 214 | 215 | zkConnect, _, err := zk.Connect([]string{g.Config().ZkHost}, time.Second*10) 216 | if err != nil { 217 | w.Write([]byte("exec fail: " + err.Error())) 218 | return 219 | } 220 | defer zkConnect.Close() 221 | 222 | var reconfigData []string 223 | reconfigData = append(reconfigData, zkNodeID) 224 | 225 | out, err := zkConnect.IncrementalReconfig(nil, reconfigData, -1) 226 | if err != nil { 227 | w.Write([]byte("exec fail: " + err.Error())) 228 | return 229 | } 230 | request, _ := json.Marshal(out) 231 | 232 | w.Write(request) 233 | } 234 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - '**' 7 | pull_request: 8 | 9 | jobs: 10 | docker-lint: 11 | runs-on: ubuntu-latest 12 | name: DockerLint 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v2 16 | - name: lint 17 | uses: brpaz/hadolint-action@master 18 | 19 | # Golang tests 20 | go-lint: 21 | runs-on: ubuntu-latest 22 | name: GoLint 23 | steps: 24 | - uses: actions/checkout@v2 25 | - name: Run golangci-lint 26 | uses: actions-contrib/golangci-lint@v1 27 | with: 28 | golangci_lint_version: 1.23 29 | args: run -v --timeout 300s 30 | 31 | go-tests: 32 | runs-on: ubuntu-latest 33 | name: go tests 34 | 35 | steps: 36 | - name: Set up Go 37 | uses: actions/setup-go@v1 38 | with: 39 | go-version: 1.14 40 | 41 | - uses: actions/checkout@v2 42 | - uses: actions/cache@v2 43 | with: 44 | path: ~/go/pkg/mod 45 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 46 | restore-keys: | 47 | ${{ runner.os }}-go- 48 | 49 | - name: make test 50 | run: | 51 | go env 52 | test -d /usr/local/kubebuilder || (curl -sSL "https://go.kubebuilder.io/dl/2.3.1/$(go env GOOS)/$(go env GOARCH)" | tar -xz -C /tmp/;sudo mv /tmp/kubebuilder_2.3.1_$(go env GOOS)_$(go env GOARCH) /usr/local/kubebuilder) 53 | make test 54 | 55 | k8s-tests: 56 | runs-on: ubuntu-latest 57 | name: K8S v${{ matrix.k8sVersion }} (CM v${{ matrix.certManager }}) 58 | 59 | strategy: 60 | fail-fast: false 61 | matrix: 62 | # https://github.com/jetstack/cert-manager/tags 63 | #certManager: ["0.15.2", "0.16.1"] 64 | certManager: ["0.16.1"] 65 | # https://snapcraft.io/microk8s 66 | #k8sVersion: ["1.17", "1.18", "1.19"] 67 | k8sVersion: ["1.19"] 68 | 69 | steps: 70 | - name: Set up Go 71 | uses: actions/setup-go@v1 72 | with: 73 | go-version: 1.14 74 | 75 | - name: Install Kubernetes v${{ matrix.k8sVersion }} 76 | run: | 77 | which kind || (curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(uname)-amd64; sudo install kind /usr/local/bin/) 78 | test -d /usr/local/kubebuilder || (curl -sSL "https://go.kubebuilder.io/dl/2.3.1/$(go env GOOS)/$(go env GOARCH)" | tar -xz -C /tmp/;sudo mv /tmp/kubebuilder_2.3.1_$(go env GOOS)_$(go env GOARCH) /usr/local/kubebuilder) 79 | cat < 0 { 160 | r.Log.Info("Allow delete quota is reached.") 161 | continue 162 | } 163 | //只允许一个健康节点进行重启 164 | _, healthy := healthyPods[candidate.Name] 165 | if maxUnavailableReached && healthy { 166 | r.Log.Info( 167 | "do_not_restart_healthy_node_if_MaxUnavailable_reached", 168 | "pod_name", candidate.Name, 169 | "zk_name", name, 170 | "namespace", namespace, 171 | ) 172 | continue 173 | } 174 | 175 | //跳过状态为terminating的节点 176 | if candidate.DeletionTimestamp != nil { 177 | r.Log.Info( 178 | "skip_already_terminating_pods", 179 | "pod_name", candidate.Name, 180 | "zk_name", name, 181 | "namespace", namespace, 182 | ) 183 | continue 184 | } 185 | 186 | delete(healthyPods, candidate.Name) 187 | podsToDelete = append(podsToDelete, candidate) 188 | allowedDeletions-- 189 | if allowedDeletions <= 0 { 190 | r.Log.Info("Allow delete quota is reached.") 191 | break 192 | } 193 | 194 | } 195 | 196 | if len(podsToDelete) == 0 { 197 | r.Log.V(1).Info( 198 | "No pod deleted during rolling upgrade", 199 | "zk_name", name, 200 | "namespace", namespace, 201 | ) 202 | return nil 203 | } 204 | 205 | for _, podToDelete := range podsToDelete { 206 | err := r.Client.Delete(&podToDelete) 207 | if err != nil { 208 | return err 209 | } 210 | deletedPods = append(deletedPods, podToDelete) 211 | } 212 | 213 | if len(deletedPods) > 0 { 214 | return errors.New("rollingupdate need requeue.") 215 | } 216 | 217 | if len(podsToDelete) > len(deletedPods) { 218 | return errors.New("rollingupdate need requeue.") 219 | } 220 | 221 | return nil 222 | } 223 | 224 | func NodesInCluster(cli *zk.BaseClient, nodeNames []string) (bool, bool, error) { 225 | ctx, cancel := context.WithTimeout(context.Background(), zk.DefaultReqTimeout) 226 | defer cancel() 227 | nodes, err := cli.GetClusterStatus(ctx) 228 | if err != nil { 229 | return false, false, err 230 | } 231 | 232 | return nodes.Mode == 1, nodes.Mode != 0, nil 233 | } 234 | 235 | // podUpgradeDone inspects the given pod and returns true if it was successfully upgraded. 236 | func podUpgradeDone(pod corev1.Pod, expectedRevision string) bool { 237 | if expectedRevision == "" { 238 | // no upgrade scheduled for the sset 239 | return false 240 | } 241 | if PodRevision(pod) != expectedRevision { 242 | // pod revision does not match the sset upgrade revision 243 | return false 244 | } 245 | return true 246 | } 247 | 248 | func PodRevision(pod corev1.Pod) string { 249 | return pod.Labels[appsv1.StatefulSetRevisionLabel] 250 | } 251 | --------------------------------------------------------------------------------