├── .gitignore ├── config ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── manifests │ ├── kustomization.yaml │ └── bases │ │ └── cockroachdb.clusterserviceversion.yaml ├── scorecard │ ├── bases │ │ └── config.yaml │ ├── patches │ │ ├── basic.config.yaml │ │ └── olm.config.yaml │ └── kustomization.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── role_binding.yaml │ ├── auth_proxy_role_binding.yaml │ ├── cluster_role.yaml │ ├── cluster_role_binding.yaml │ ├── leader_election_role_binding.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_service.yaml │ ├── cockroachdb_viewer_role.yaml │ ├── leader_election_role.yaml │ ├── kustomization.yaml │ ├── cockroachdb_editor_role.yaml │ └── role.yaml ├── samples │ ├── kustomization.yaml │ └── charts.operatorhub.io_v1alpha1_cockroachdb.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── crd │ ├── kustomization.yaml │ └── bases │ │ └── charts.operatorhub.io_cockroachdbs.yaml └── default │ ├── kustomization.yaml │ └── manager_auth_proxy_patch.yaml ├── Dockerfile ├── PROJECT ├── watches.yaml ├── bundle ├── manifests │ ├── cockroachdb-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml │ ├── cockroachdb-controller-manager-metrics-service_v1_service.yaml │ └── charts.operatorhub.io_cockroachdbs.yaml ├── metadata │ └── annotations.yaml └── tests │ └── scorecard │ └── config.yaml ├── helm-charts └── cockroachdb │ ├── Chart.yaml │ ├── templates │ ├── serviceaccount.yaml │ ├── clusterrole.yaml │ ├── role.yaml │ ├── rolebinding.yaml │ ├── poddisruptionbudget.yaml │ ├── clusterrolebinding.yaml │ ├── secret.registry.yaml │ ├── serviceMonitor.yaml │ ├── ingress.yaml │ ├── service.public.yaml │ ├── tests │ │ └── client.yaml │ ├── _helpers.tpl │ ├── NOTES.txt │ ├── networkpolicy.yaml │ ├── service.discovery.yaml │ ├── job.init.yaml │ └── statefulset.yaml │ ├── CONTRIBUTING.md │ ├── values.yaml │ └── README.md ├── bundle.Dockerfile ├── Makefile └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/manifests/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../default 3 | - ../samples 4 | - ../scorecard 5 | -------------------------------------------------------------------------------- /config/scorecard/bases/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scorecard.operatorframework.io/v1alpha3 2 | kind: Configuration 3 | metadata: 4 | name: config 5 | stages: 6 | - parallel: true 7 | tests: [] 8 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: ["/metrics"] 7 | verbs: ["get"] 8 | -------------------------------------------------------------------------------- /config/samples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | ## Append samples you want in your CSV to this file as resources ## 2 | resources: 3 | - charts.operatorhub.io_v1alpha1_cockroachdb.yaml 4 | # +kubebuilder:scaffold:manifestskustomizesamples 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM quay.io/operator-framework/helm-operator:v1.3.0 3 | 4 | ENV HOME=/opt/helm 5 | COPY watches.yaml ${HOME}/watches.yaml 6 | COPY helm-charts ${HOME}/helm-charts 7 | WORKDIR ${HOME} 8 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: charts.operatorhub.io 2 | layout: helm.sdk.operatorframework.io/v1 3 | projectName: cockroachdb 4 | resources: 5 | - group: charts.operatorhub.io 6 | kind: Cockroachdb 7 | version: v1alpha1 8 | version: 3-alpha -------------------------------------------------------------------------------- /watches.yaml: -------------------------------------------------------------------------------- 1 | # Use the 'create api' subcommand to add watches to this file. 2 | - group: charts.operatorhub.io 3 | version: v1alpha1 4 | kind: Cockroachdb 5 | chart: helm-charts/cockroachdb 6 | # +kubebuilder:scaffold:watch 7 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | images: 6 | - name: controller 7 | newName: quay.io/helmoperators/cockroachdb 8 | newTag: v5.0.4 9 | -------------------------------------------------------------------------------- /config/scorecard/patches/basic.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - basic-check-spec 7 | image: quay.io/operator-framework/scorecard-test:v1.3.0 8 | labels: 9 | suite: basic 10 | test: basic-check-spec-test 11 | -------------------------------------------------------------------------------- /bundle/manifests/cockroachdb-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | creationTimestamp: null 5 | name: cockroachdb-metrics-reader 6 | rules: 7 | - nonResourceURLs: 8 | - /metrics 9 | verbs: 10 | - get 11 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/charts.operatorhub.io_cockroachdbs.yaml 6 | # +kubebuilder:scaffold:crdkustomizeresource 7 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/cluster_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: manager-role 5 | rules: 6 | - apiGroups: 7 | - rbac.authorization.k8s.io 8 | resources: 9 | - roles 10 | - clusterroles 11 | - rolebindings 12 | - clusterrolebindings 13 | verbs: 14 | - '*' 15 | -------------------------------------------------------------------------------- /config/rbac/cluster_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: ["authentication.k8s.io"] 7 | resources: 8 | - tokenreviews 9 | verbs: ["create"] 10 | - apiGroups: ["authorization.k8s.io"] 11 | resources: 12 | - subjectaccessreviews 13 | verbs: ["create"] 14 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | selector: 15 | matchLabels: 16 | control-plane: controller-manager 17 | -------------------------------------------------------------------------------- /bundle/manifests/cockroachdb-controller-manager-metrics-service_v1_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | control-plane: controller-manager 7 | name: cockroachdb-controller-manager-metrics-service 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | status: 16 | loadBalancer: {} 17 | -------------------------------------------------------------------------------- /config/rbac/cockroachdb_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view cockroachdbs. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: cockroachdb-viewer-role 6 | rules: 7 | - apiGroups: 8 | - charts.operatorhub.io 9 | resources: 10 | - cockroachdbs 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - charts.operatorhub.io 17 | resources: 18 | - cockroachdbs/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - events 23 | verbs: 24 | - create 25 | - patch 26 | -------------------------------------------------------------------------------- /config/scorecard/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - bases/config.yaml 3 | patchesJson6902: 4 | - path: patches/basic.config.yaml 5 | target: 6 | group: scorecard.operatorframework.io 7 | version: v1alpha3 8 | kind: Configuration 9 | name: config 10 | - path: patches/olm.config.yaml 11 | target: 12 | group: scorecard.operatorframework.io 13 | version: v1alpha3 14 | kind: Configuration 15 | name: config 16 | # +kubebuilder:scaffold:patchesJson6902 17 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 20.2.4 3 | description: CockroachDB is a scalable, survivable, strongly-consistent SQL database. 4 | home: https://www.cockroachlabs.com 5 | icon: https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png 6 | maintainers: 7 | - email: helm-charts@cockroachlabs.com 8 | name: cockroachlabs 9 | name: cockroachdb 10 | sources: 11 | - https://github.com/cockroachdb/cockroach 12 | version: 5.0.4 13 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - role.yaml 3 | - role_binding.yaml 4 | - cluster_role.yaml 5 | - cluster_role_binding.yaml 6 | - leader_election_role.yaml 7 | - leader_election_role_binding.yaml 8 | # Comment the following 4 lines if you want to disable 9 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 10 | # which protects your /metrics endpoint. 11 | - auth_proxy_service.yaml 12 | - auth_proxy_role.yaml 13 | - auth_proxy_role_binding.yaml 14 | - auth_proxy_client_clusterrole.yaml 15 | -------------------------------------------------------------------------------- /config/rbac/cockroachdb_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit cockroachdbs. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: cockroachdb-editor-role 6 | rules: 7 | - apiGroups: 8 | - charts.operatorhub.io 9 | resources: 10 | - cockroachdbs 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - charts.operatorhub.io 21 | resources: 22 | - cockroachdbs/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.tls.enabled .Values.tls.serviceAccount.create }} 2 | kind: ServiceAccount 3 | apiVersion: v1 4 | metadata: 5 | name: {{ template "cockroachdb.tls.serviceAccount.name" . }} 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | {{- end }} 16 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcome! 4 | 5 | For every change, please increment the `version` contained in 6 | [Chart.yaml](https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/Chart.yaml). 7 | The `version` roughly follows the [SEMVER](https://semver.org/) versioning 8 | pattern. For changes which do not affect backwards compatibility, the PATCH or 9 | MINOR version must be incremented, e.g. `4.1.3` -> `4.1.4`. For changes which 10 | affect the backwards compatibility of the chart, the major version must be 11 | incremented, e.g. `4.1.3` -> `5.0.0`. Examples of changes which affect backwards 12 | compatibility include any major version releases of CockroachDB, as well as any 13 | breaking changes to the CockroachDB chart templates. 14 | 15 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.tls.enabled (not .Values.tls.certs.provided) }} 2 | kind: ClusterRole 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: {{ template "cockroachdb.fullname" . }} 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | rules: 16 | - apiGroups: ["certificates.k8s.io"] 17 | resources: ["certificatesigningrequests"] 18 | verbs: ["create", "get", "watch"] 19 | {{- end }} -------------------------------------------------------------------------------- /bundle/metadata/annotations.yaml: -------------------------------------------------------------------------------- 1 | annotations: 2 | operators.operatorframework.io.bundle.channel.default.v1: stable-5.x 3 | operators.operatorframework.io.bundle.channels.v1: stable-5.x 4 | operators.operatorframework.io.bundle.manifests.v1: manifests/ 5 | operators.operatorframework.io.bundle.mediatype.v1: registry+v1 6 | operators.operatorframework.io.bundle.metadata.v1: metadata/ 7 | operators.operatorframework.io.bundle.package.v1: cockroachdb 8 | operators.operatorframework.io.metrics.builder: operator-sdk-v1.3.0 9 | operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 10 | operators.operatorframework.io.metrics.project_layout: helm.sdk.operatorframework.io/v1 11 | operators.operatorframework.io.test.config.v1: tests/scorecard/ 12 | operators.operatorframework.io.test.mediatype.v1: scorecard+v1 13 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/role.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.tls.enabled }} 2 | kind: Role 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: {{ template "cockroachdb.fullname" . }} 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | rules: 16 | - apiGroups: [""] 17 | resources: ["secrets"] 18 | {{- if .Values.tls.certs.provided }} 19 | verbs: ["get"] 20 | {{- else }} 21 | verbs: ["create", "get"] 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.tls.enabled }} 2 | kind: RoleBinding 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: {{ template "cockroachdb.fullname" . }} 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | roleRef: 16 | apiGroup: rbac.authorization.k8s.io 17 | kind: Role 18 | name: {{ template "cockroachdb.fullname" . }} 19 | subjects: 20 | - kind: ServiceAccount 21 | name: {{ template "cockroachdb.tls.serviceAccount.name" . }} 22 | namespace: {{ .Release.Namespace | quote }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: cockroachdb-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: cockroachdb- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 20 | #- ../prometheus 21 | 22 | patchesStrategicMerge: 23 | # Protect the /metrics endpoint by putting it behind auth. 24 | # If you want your controller-manager to expose the /metrics 25 | # endpoint w/o any authn/z, please comment the following line. 26 | - manager_auth_proxy_patch.yaml 27 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | name: https 22 | - name: manager 23 | args: 24 | - "--metrics-addr=127.0.0.1:8080" 25 | - "--enable-leader-election" 26 | - "--leader-election-id=cockroachdb" 27 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/poddisruptionbudget.yaml: -------------------------------------------------------------------------------- 1 | kind: PodDisruptionBudget 2 | apiVersion: policy/v1beta1 3 | metadata: 4 | name: {{ template "cockroachdb.fullname" . }}-budget 5 | namespace: {{ .Release.Namespace | quote }} 6 | labels: 7 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 8 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 9 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 10 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 11 | {{- with .Values.labels }} 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | spec: 15 | selector: 16 | matchLabels: 17 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 18 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 19 | {{- with .Values.statefulset.labels }} 20 | {{- toYaml . | nindent 6 }} 21 | {{- end }} 22 | maxUnavailable: {{ .Values.statefulset.budget.maxUnavailable | int64 }} 23 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.tls.enabled (not .Values.tls.certs.provided) }} 2 | kind: ClusterRoleBinding 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: {{ template "cockroachdb.fullname" . }} 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | roleRef: 16 | apiGroup: rbac.authorization.k8s.io 17 | kind: ClusterRole 18 | name: {{ template "cockroachdb.fullname" . }} 19 | subjects: 20 | - kind: ServiceAccount 21 | name: {{ template "cockroachdb.tls.serviceAccount.name" . }} 22 | namespace: {{ .Release.Namespace | quote }} 23 | {{- end }} -------------------------------------------------------------------------------- /bundle.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | 3 | LABEL operators.operatorframework.io.bundle.mediatype.v1=registry+v1 4 | LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ 5 | LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ 6 | LABEL operators.operatorframework.io.bundle.package.v1=cockroachdb 7 | LABEL operators.operatorframework.io.bundle.channels.v1=stable-5.x 8 | LABEL operators.operatorframework.io.bundle.channel.default.v1=stable-5.x 9 | LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.3.0 10 | LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 11 | LABEL operators.operatorframework.io.metrics.project_layout=helm.sdk.operatorframework.io/v1 12 | LABEL operators.operatorframework.io.test.config.v1=tests/scorecard/ 13 | LABEL operators.operatorframework.io.test.mediatype.v1=scorecard+v1 14 | 15 | COPY bundle/manifests /manifests/ 16 | COPY bundle/metadata /metadata/ 17 | COPY bundle/tests/scorecard /tests/scorecard/ 18 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/secret.registry.yaml: -------------------------------------------------------------------------------- 1 | {{- range $name, $cred := dict "db" (.Values.image.credentials) "init-certs" (.Values.tls.init.image.credentials) }} 2 | {{- if not (empty $cred) }} 3 | {{- if or (and (eq $name "init-certs") $.Values.tls.enabled) (ne $name "init-certs") }} 4 | --- 5 | kind: Secret 6 | apiVersion: v1 7 | metadata: 8 | name: {{ template "cockroachdb.fullname" $ }}.{{ $name }}.registry 9 | namespace: {{ $.Release.Namespace | quote }} 10 | labels: 11 | helm.sh/chart: {{ template "cockroachdb.chart" $ }} 12 | app.kubernetes.io/name: {{ template "cockroachdb.name" $ }} 13 | app.kubernetes.io/instance: {{ $.Release.Name | quote }} 14 | app.kubernetes.io/managed-by: {{ $.Release.Service | quote }} 15 | {{- with $.Values.labels }} 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | type: kubernetes.io/dockerconfigjson 19 | data: 20 | .dockerconfigjson: {{ printf `{"auths":{%s:{"auth":"%s"}}}` ($cred.registry | quote) (printf "%s:%s" $cred.username $cred.password | b64enc) | b64enc | quote }} 21 | {{- end }} 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /config/scorecard/patches/olm.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - olm-bundle-validation 7 | image: quay.io/operator-framework/scorecard-test:v1.3.0 8 | labels: 9 | suite: olm 10 | test: olm-bundle-validation-test 11 | - op: add 12 | path: /stages/0/tests/- 13 | value: 14 | entrypoint: 15 | - scorecard-test 16 | - olm-crds-have-validation 17 | image: quay.io/operator-framework/scorecard-test:v1.3.0 18 | labels: 19 | suite: olm 20 | test: olm-crds-have-validation-test 21 | - op: add 22 | path: /stages/0/tests/- 23 | value: 24 | entrypoint: 25 | - scorecard-test 26 | - olm-crds-have-resources 27 | image: quay.io/operator-framework/scorecard-test:v1.3.0 28 | labels: 29 | suite: olm 30 | test: olm-crds-have-resources-test 31 | - op: add 32 | path: /stages/0/tests/- 33 | value: 34 | entrypoint: 35 | - scorecard-test 36 | - olm-spec-descriptors 37 | image: quay.io/operator-framework/scorecard-test:v1.3.0 38 | labels: 39 | suite: olm 40 | test: olm-spec-descriptors-test 41 | - op: add 42 | path: /stages/0/tests/- 43 | value: 44 | entrypoint: 45 | - scorecard-test 46 | - olm-status-descriptors 47 | image: quay.io/operator-framework/scorecard-test:v1.3.0 48 | labels: 49 | suite: olm 50 | test: olm-status-descriptors-test 51 | -------------------------------------------------------------------------------- /bundle/tests/scorecard/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scorecard.operatorframework.io/v1alpha3 2 | kind: Configuration 3 | metadata: 4 | name: config 5 | stages: 6 | - parallel: true 7 | tests: 8 | - entrypoint: 9 | - scorecard-test 10 | - basic-check-spec 11 | image: quay.io/operator-framework/scorecard-test:v1.3.0 12 | labels: 13 | suite: basic 14 | test: basic-check-spec-test 15 | - entrypoint: 16 | - scorecard-test 17 | - olm-bundle-validation 18 | image: quay.io/operator-framework/scorecard-test:v1.3.0 19 | labels: 20 | suite: olm 21 | test: olm-bundle-validation-test 22 | - entrypoint: 23 | - scorecard-test 24 | - olm-crds-have-validation 25 | image: quay.io/operator-framework/scorecard-test:v1.3.0 26 | labels: 27 | suite: olm 28 | test: olm-crds-have-validation-test 29 | - entrypoint: 30 | - scorecard-test 31 | - olm-crds-have-resources 32 | image: quay.io/operator-framework/scorecard-test:v1.3.0 33 | labels: 34 | suite: olm 35 | test: olm-crds-have-resources-test 36 | - entrypoint: 37 | - scorecard-test 38 | - olm-spec-descriptors 39 | image: quay.io/operator-framework/scorecard-test:v1.3.0 40 | labels: 41 | suite: olm 42 | test: olm-spec-descriptors-test 43 | - entrypoint: 44 | - scorecard-test 45 | - olm-status-descriptors 46 | image: quay.io/operator-framework/scorecard-test:v1.3.0 47 | labels: 48 | suite: olm 49 | test: olm-status-descriptors-test 50 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: manager-role 5 | rules: 6 | ## 7 | ## Base operator rules 8 | ## 9 | # We need to get namespaces so the operator can read namespaces to ensure they exist 10 | - apiGroups: 11 | - "" 12 | resources: 13 | - namespaces 14 | verbs: 15 | - get 16 | # We need to manage Helm release secrets 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - secrets 21 | - configmaps 22 | - serviceaccounts 23 | verbs: 24 | - "*" 25 | # We need to create events on CRs about things happening during reconciliation 26 | - apiGroups: 27 | - "" 28 | resources: 29 | - events 30 | verbs: 31 | - create 32 | 33 | ## 34 | ## Rules for charts.operatorhub.io/v1alpha1, Kind: Cockroachdb 35 | ## 36 | - apiGroups: 37 | - charts.operatorhub.io 38 | resources: 39 | - cockroachdbs 40 | - cockroachdbs/status 41 | - cockroachdbs/finalizers 42 | verbs: 43 | - create 44 | - delete 45 | - get 46 | - list 47 | - patch 48 | - update 49 | - watch 50 | - verbs: 51 | - "*" 52 | apiGroups: 53 | - "apps" 54 | resources: 55 | - "statefulsets" 56 | - verbs: 57 | - "*" 58 | apiGroups: 59 | - "batch" 60 | resources: 61 | - "jobs" 62 | - verbs: 63 | - "*" 64 | apiGroups: 65 | - "policy" 66 | resources: 67 | - "poddisruptionbudgets" 68 | - verbs: 69 | - "*" 70 | apiGroups: 71 | - "" 72 | resources: 73 | - "services" 74 | - "pods" 75 | - verbs: 76 | - "*" 77 | apiGroups: 78 | - "monitoring.coreos.com" 79 | resources: 80 | - "servicemonitors" 81 | - verbs: 82 | - "*" 83 | apiGroups: 84 | - "networking.k8s.io" 85 | resources: 86 | - "networkpolicies" 87 | 88 | # +kubebuilder:scaffold:rules 89 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/serviceMonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- $serviceMonitor := .Values.serviceMonitor -}} 2 | {{- $ports := .Values.service.ports -}} 3 | {{- if $serviceMonitor.enabled }} 4 | apiVersion: monitoring.coreos.com/v1 5 | kind: ServiceMonitor 6 | metadata: 7 | name: {{ template "cockroachdb.fullname" . }} 8 | namespace: {{ .Release.Namespace | quote }} 9 | labels: 10 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 11 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 14 | {{- if $serviceMonitor.labels }} 15 | {{ toYaml $serviceMonitor.labels | nindent 4 }} 16 | {{- end }} 17 | {{- with .Values.labels }} 18 | {{- toYaml . | nindent 4 }} 19 | {{- end }} 20 | {{- if $serviceMonitor.annotations }} 21 | annotations: 22 | {{ toYaml $serviceMonitor.annotations | nindent 4 }} 23 | {{- end }} 24 | spec: 25 | selector: 26 | matchLabels: 27 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 28 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 29 | {{- with .Values.service.discovery.labels }} 30 | {{- toYaml . | nindent 6 }} 31 | {{- end }} 32 | {{- with .Values.labels }} 33 | {{- toYaml . | nindent 6 }} 34 | {{- end }} 35 | namespaceSelector: 36 | any: true 37 | endpoints: 38 | - port: {{ $ports.http.name | quote }} 39 | path: /_status/vars 40 | {{- if $serviceMonitor.interval }} 41 | interval: {{ $serviceMonitor.interval }} 42 | {{- end }} 43 | {{- if $serviceMonitor.scrapeTimeout }} 44 | scrapeTimeout: {{ $serviceMonitor.scrapeTimeout }} 45 | {{- end }} 46 | {{- end }} -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $paths := .Values.ingress.paths -}} 3 | {{- $ports := .Values.service.ports -}} 4 | {{- $fullName := include "cockroachdb.fullname" . -}} 5 | apiVersion: networking.k8s.io/v1beta1 6 | kind: Ingress 7 | metadata: 8 | {{- if .Values.ingress.annotations }} 9 | annotations: 10 | {{- range $key, $value := .Values.ingress.annotations }} 11 | {{ $key }}: {{ $value | quote }} 12 | {{- end }} 13 | {{- end }} 14 | name: {{ $fullName }}-ingress 15 | namespace: {{ .Release.Namespace }} 16 | labels: 17 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 18 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 19 | app.kubernetes.io/instance: {{ $.Release.Name | quote }} 20 | app.kubernetes.io/managed-by: {{ $.Release.Service | quote }} 21 | {{- if .Values.ingress.labels }} 22 | {{- toYaml .Values.ingress.labels | nindent 4 }} 23 | {{- end }} 24 | spec: 25 | rules: 26 | {{- if .Values.ingress.hosts }} 27 | {{- range $host := .Values.ingress.hosts }} 28 | - host: {{ $host }} 29 | http: 30 | paths: 31 | {{- range $path := $paths }} 32 | - path: {{ $path }} 33 | backend: 34 | serviceName: {{ $fullName }}-public 35 | servicePort: {{ $ports.http.name | quote }} 36 | {{- end -}} 37 | {{- end -}} 38 | {{- else }} 39 | - http: 40 | paths: 41 | {{- range $path := $paths }} 42 | - path: {{ $path }} 43 | backend: 44 | serviceName: {{ $fullName }}-public 45 | servicePort: {{ $ports.http.name | quote }} 46 | {{- end -}} 47 | {{- end -}} 48 | {{- if .Values.ingress.tls }} 49 | tls: 50 | {{- toYaml .Values.ingress.tls | nindent 4 }} 51 | {{- end -}} 52 | {{- end -}} -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | labels: 23 | control-plane: controller-manager 24 | spec: 25 | containers: 26 | - env: 27 | - name: WATCH_NAMESPACE 28 | valueFrom: 29 | fieldRef: 30 | fieldPath: metadata.annotations['olm.targetNamespaces'] 31 | - name: POD_NAME 32 | valueFrom: 33 | fieldRef: 34 | fieldPath: metadata.name 35 | - name: OPERATOR_NAME 36 | value: cockroachdb 37 | # - name: COCKROACH_IMAGE_REPOSITORY 38 | # value: docker.io/cockroachdb/cockroach 39 | # - name: COCKROACH_IMAGE_TAG 40 | # value: v19.2.5 41 | # - name: COCKROACH_TLS_IMAGE_REPOSITORY 42 | # value: docker.io/cockroachdb/cockroach-k8s-request-cert 43 | # - name: COCKROACH_TLS_IMAGE_TAG 44 | # value: "0.4" 45 | # image: quay.io/helmoperators/cockroachdb@sha256:fa7263b3e18f8f7d6b6c8f8105782861712be5655058ec9bc2002f85078c8e32 46 | image: controller:latest 47 | args: 48 | - "--enable-leader-election" 49 | - "--leader-election-id=cockroachdb" 50 | name: manager 51 | resources: 52 | limits: 53 | cpu: 100m 54 | memory: 90Mi 55 | requests: 56 | cpu: 100m 57 | memory: 60Mi 58 | terminationGracePeriodSeconds: 10 59 | -------------------------------------------------------------------------------- /config/crd/bases/charts.operatorhub.io_cockroachdbs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: cockroachdbs.charts.operatorhub.io 6 | spec: 7 | group: charts.operatorhub.io 8 | names: 9 | kind: Cockroachdb 10 | listKind: CockroachdbList 11 | plural: cockroachdbs 12 | singular: cockroachdb 13 | scope: Namespaced 14 | versions: 15 | - name: v1alpha1 16 | schema: 17 | openAPIV3Schema: 18 | description: Cockroachdb is the Schema for the cockroachdbs API 19 | properties: 20 | apiVersion: 21 | description: 'APIVersion defines the versioned schema of this representation 22 | of an object. Servers should convert recognized schemas to the latest 23 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 24 | type: string 25 | kind: 26 | description: 'Kind is a string value representing the REST resource this 27 | object represents. Servers may infer this from the endpoint the client 28 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 29 | type: string 30 | metadata: 31 | type: object 32 | spec: 33 | description: Spec defines the desired state of Cockroachdb 34 | type: object 35 | x-kubernetes-preserve-unknown-fields: true 36 | status: 37 | description: Status defines the observed state of Cockroachdb 38 | type: object 39 | x-kubernetes-preserve-unknown-fields: true 40 | type: object 41 | served: true 42 | storage: true 43 | subresources: 44 | status: {} 45 | -------------------------------------------------------------------------------- /bundle/manifests/charts.operatorhub.io_cockroachdbs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | creationTimestamp: null 5 | name: cockroachdbs.charts.operatorhub.io 6 | spec: 7 | group: charts.operatorhub.io 8 | names: 9 | kind: Cockroachdb 10 | listKind: CockroachdbList 11 | plural: cockroachdbs 12 | singular: cockroachdb 13 | scope: Namespaced 14 | versions: 15 | - name: v1alpha1 16 | schema: 17 | openAPIV3Schema: 18 | description: Cockroachdb is the Schema for the cockroachdbs API 19 | properties: 20 | apiVersion: 21 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 22 | type: string 23 | kind: 24 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 25 | type: string 26 | metadata: 27 | type: object 28 | spec: 29 | description: Spec defines the desired state of Cockroachdb 30 | type: object 31 | x-kubernetes-preserve-unknown-fields: true 32 | status: 33 | description: Status defines the observed state of Cockroachdb 34 | type: object 35 | x-kubernetes-preserve-unknown-fields: true 36 | type: object 37 | served: true 38 | storage: true 39 | subresources: 40 | status: {} 41 | status: 42 | acceptedNames: 43 | kind: "" 44 | plural: "" 45 | conditions: null 46 | storedVersions: null 47 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/service.public.yaml: -------------------------------------------------------------------------------- 1 | # This Service is meant to be used by clients of the database. 2 | # It exposes a ClusterIP that will automatically load balance connections 3 | # to the different database Pods. 4 | kind: Service 5 | apiVersion: v1 6 | metadata: 7 | name: {{ template "cockroachdb.fullname" . }}-public 8 | namespace: {{ .Release.Namespace | quote }} 9 | labels: 10 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 11 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 14 | {{- with .Values.service.public.labels }} 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | {{- with .Values.labels }} 18 | {{- toYaml . | nindent 4 }} 19 | {{- end }} 20 | {{- with .Values.service.public.annotations }} 21 | annotations: {{- toYaml . | nindent 4 }} 22 | {{- end }} 23 | spec: 24 | type: {{ .Values.service.public.type | quote }} 25 | ports: 26 | {{- $ports := .Values.service.ports }} 27 | # The main port, served by gRPC, serves Postgres-flavor SQL, inter-node 28 | # traffic and the CLI. 29 | - name: {{ $ports.grpc.external.name | quote }} 30 | port: {{ $ports.grpc.external.port | int64 }} 31 | targetPort: grpc 32 | {{- if ne ($ports.grpc.internal.port | int64) ($ports.grpc.external.port | int64) }} 33 | - name: {{ $ports.grpc.internal.name | quote }} 34 | port: {{ $ports.grpc.internal.port | int64 }} 35 | targetPort: grpc 36 | {{- end }} 37 | # The secondary port serves the UI as well as health and debug endpoints. 38 | - name: {{ $ports.http.name | quote }} 39 | port: {{ $ports.http.port | int64 }} 40 | targetPort: http 41 | selector: 42 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 43 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 44 | {{- with .Values.statefulset.labels }} 45 | {{- toYaml . | nindent 4 }} 46 | {{- end }} 47 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/tests/client.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: {{ template "cockroachdb.fullname" . }}-test 5 | namespace: {{ .Release.Namespace | quote }} 6 | {{- if .Values.networkPolicy.enabled }} 7 | labels: 8 | {{ template "cockroachdb.fullname" . }}-client: "true" 9 | {{- end }} 10 | annotations: 11 | helm.sh/hook: test-success 12 | spec: 13 | restartPolicy: Never 14 | {{- if .Values.image.credentials }} 15 | imagePullSecrets: 16 | - name: {{ template "cockroachdb.fullname" . }}.db.registry 17 | {{- end }} 18 | {{- if .Values.tls.certs.provided }} 19 | volumes: 20 | - name: client-certs 21 | {{- if .Values.tls.certs.tlsSecret }} 22 | projected: 23 | sources: 24 | - secret: 25 | name: {{ .Values.tls.certs.clientRootSecret }} 26 | items: 27 | - key: ca.crt 28 | path: ca.crt 29 | mode: 0400 30 | - key: tls.crt 31 | path: client.root.crt 32 | mode: 0400 33 | - key: tls.key 34 | path: client.root.key 35 | mode: 0400 36 | {{- else }} 37 | secret: 38 | secretName: {{ .Values.tls.certs.clientRootSecret }} 39 | defaultMode: 0400 40 | {{- end }} 41 | {{- end }} 42 | containers: 43 | - name: client-test 44 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 45 | imagePullPolicy: {{ .Values.image.pullPolicy | quote }} 46 | {{- if .Values.tls.certs.provided }} 47 | volumeMounts: 48 | - name: client-certs 49 | mountPath: /cockroach-certs 50 | {{- end }} 51 | command: 52 | - /cockroach/cockroach 53 | - sql 54 | {{- if .Values.tls.certs.provided }} 55 | - --certs-dir 56 | - /cockroach-certs 57 | {{- else }} 58 | - --insecure 59 | {{- end}} 60 | - --host 61 | - {{ template "cockroachdb.fullname" . }}-public.{{ .Release.Namespace }} 62 | - --port 63 | - {{ .Values.service.ports.grpc.external.port | quote }} 64 | - -e 65 | - SHOW DATABASES; 66 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "cockroachdb.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 56 | trimSuffix "-" -}} 6 | {{- end -}} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "cockroachdb.fullname" -}} 14 | {{- if .Values.fullnameOverride -}} 15 | {{- .Values.fullnameOverride | trunc 56 | trimSuffix "-" -}} 16 | {{- else -}} 17 | {{- $name := default .Chart.Name .Values.nameOverride -}} 18 | {{- if contains $name .Release.Name -}} 19 | {{- .Release.Name | trunc 56 | trimSuffix "-" -}} 20 | {{- else -}} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 56 | trimSuffix "-" -}} 22 | {{- end -}} 23 | {{- end -}} 24 | {{- end -}} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "cockroachdb.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 56 | trimSuffix "-" -}} 31 | {{- end -}} 32 | 33 | {{/* 34 | Create the name of the ServiceAccount to use. 35 | */}} 36 | {{- define "cockroachdb.tls.serviceAccount.name" -}} 37 | {{- if .Values.tls.serviceAccount.create -}} 38 | {{- default (include "cockroachdb.fullname" .) .Values.tls.serviceAccount.name -}} 39 | {{- else -}} 40 | {{- default "default" .Values.tls.serviceAccount.name -}} 41 | {{- end -}} 42 | {{- end -}} 43 | 44 | {{/* 45 | Return the appropriate apiVersion for NetworkPolicy. 46 | */}} 47 | {{- define "cockroachdb.networkPolicy.apiVersion" -}} 48 | {{- if semverCompare ">=1.4-0, <=1.7-0" .Capabilities.KubeVersion.GitVersion -}} 49 | {{- print "extensions/v1beta1" -}} 50 | {{- else if semverCompare "^1.7-0" .Capabilities.KubeVersion.GitVersion -}} 51 | {{- print "networking.k8s.io/v1" -}} 52 | {{- end -}} 53 | {{- end -}} 54 | 55 | {{/* 56 | Return the appropriate apiVersion for StatefulSets 57 | */}} 58 | {{- define "cockroachdb.statefulset.apiVersion" -}} 59 | {{- if semverCompare "<1.12-0" .Capabilities.KubeVersion.GitVersion -}} 60 | {{- print "apps/v1beta1" -}} 61 | {{- else -}} 62 | {{- print "apps/v1" -}} 63 | {{- end -}} 64 | {{- end -}} 65 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | CockroachDB can be accessed via port {{ .Values.service.ports.grpc.external.port }} at the 2 | following DNS name from within your cluster: 3 | 4 | {{ template "cockroachdb.fullname" . }}-public.{{ .Release.Namespace }}.svc.cluster.local 5 | 6 | Because CockroachDB supports the PostgreSQL wire protocol, you can connect to 7 | the cluster using any available PostgreSQL client. 8 | 9 | {{- if not .Values.tls.enabled }} 10 | 11 | For example, you can open up a SQL shell to the cluster by running: 12 | 13 | kubectl run -it --rm cockroach-client \ 14 | --image=cockroachdb/cockroach \ 15 | --restart=Never \ 16 | {{- if .Values.networkPolicy.enabled }} 17 | --labels="{{ template "cockroachdb.fullname" . }}-client=true" \ 18 | {{- end }} 19 | --command -- \ 20 | ./cockroach sql --insecure --host={{ template "cockroachdb.fullname" . }}-public.{{ .Release.Namespace }} 21 | 22 | From there, you can interact with the SQL shell as you would any other SQL 23 | shell, confident that any data you write will be safe and available even if 24 | parts of your cluster fail. 25 | {{- else }} 26 | 27 | Note that because the cluster is running in secure mode, any client application 28 | that you attempt to connect will either need to have a valid client certificate 29 | or a valid username and password. 30 | {{- end }} 31 | 32 | {{- if and (.Values.networkPolicy.enabled) (not (empty .Values.networkPolicy.ingress.grpc)) }} 33 | 34 | Note: Since NetworkPolicy is enabled, the only Pods allowed to connect to this 35 | CockroachDB cluster are: 36 | 37 | 1. Having the label: "{{ template "cockroachdb.fullname" . }}-client=true" 38 | 39 | 2. Matching the following rules: {{- toYaml .Values.networkPolicy.ingress.grpc | nindent 0 }} 40 | {{- end }} 41 | 42 | Finally, to open up the CockroachDB admin UI, you can port-forward from your 43 | local machine into one of the instances in the cluster: 44 | 45 | kubectl port-forward {{ template "cockroachdb.fullname" . }}-0 {{ index .Values.conf `http-port` | int64 }} 46 | 47 | Then you can access the admin UI at http{{ if .Values.tls.enabled }}s{{ end }}://localhost:{{ index .Values.conf `http-port` | int64 }}/ in your web browser. 48 | 49 | For more information on using CockroachDB, please see the project's docs at: 50 | https://www.cockroachlabs.com/docs/ 51 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/networkpolicy.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.networkPolicy.enabled }} 2 | kind: NetworkPolicy 3 | apiVersion: {{ template "cockroachdb.networkPolicy.apiVersion" . }} 4 | metadata: 5 | name: {{ template "cockroachdb.tls.serviceAccount.name" . }} 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | spec: 16 | podSelector: 17 | matchLabels: 18 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 19 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 20 | {{- with .Values.statefulset.labels }} 21 | {{- toYaml . | nindent 6 }} 22 | {{- end }} 23 | ingress: 24 | - ports: 25 | - port: grpc 26 | {{- with .Values.networkPolicy.ingress.grpc }} 27 | from: 28 | # Allow connections via custom rules. 29 | {{- toYaml . | nindent 8 }} 30 | # Allow client connection via pre-considered label. 31 | - podSelector: 32 | matchLabels: 33 | {{ template "cockroachdb.fullname" . }}-client: "true" 34 | # Allow other CockroachDBs to connect to form a cluster. 35 | - podSelector: 36 | matchLabels: 37 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 38 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 39 | {{- with .Values.statefulset.labels }} 40 | {{- toYaml . | nindent 14 }} 41 | {{- end }} 42 | {{- if gt (.Values.statefulset.replicas | int64) 1 }} 43 | # Allow init Job to connect to bootstrap a cluster. 44 | - podSelector: 45 | matchLabels: 46 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 47 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 48 | {{- with .Values.init.labels }} 49 | {{- toYaml . | nindent 14 }} 50 | {{- end }} 51 | {{- end }} 52 | {{- end }} 53 | # Allow connections to admin UI and for Prometheus. 54 | - ports: 55 | - port: http 56 | {{- with .Values.networkPolicy.ingress.http }} 57 | from: {{- toYaml . | nindent 8 }} 58 | {{- end }} 59 | {{- end }} 60 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/service.discovery.yaml: -------------------------------------------------------------------------------- 1 | # This service only exists to create DNS entries for each pod in 2 | # the StatefulSet such that they can resolve each other's IP addresses. 3 | # It does not create a load-balanced ClusterIP and should not be used directly 4 | # by clients in most circumstances. 5 | kind: Service 6 | apiVersion: v1 7 | metadata: 8 | name: {{ template "cockroachdb.fullname" . }} 9 | namespace: {{ .Release.Namespace | quote }} 10 | labels: 11 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 12 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 13 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 14 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 15 | {{- with .Values.service.discovery.labels }} 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | {{- with .Values.labels }} 19 | {{- toYaml . | nindent 4 }} 20 | {{- end }} 21 | annotations: 22 | # Use this annotation in addition to the actual field below because the 23 | # annotation will stop being respected soon, but the field is broken in 24 | # some versions of Kubernetes: 25 | # https://github.com/kubernetes/kubernetes/issues/58662 26 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 27 | # Enable automatic monitoring of all instances when Prometheus is running 28 | # in the cluster. 29 | prometheus.io/scrape: "true" 30 | prometheus.io/path: _status/vars 31 | prometheus.io/port: {{ .Values.service.ports.http.port | quote }} 32 | {{- with .Values.service.discovery.annotations }} 33 | {{- toYaml . | nindent 4 }} 34 | {{- end }} 35 | spec: 36 | clusterIP: None 37 | # We want all Pods in the StatefulSet to have their addresses published for 38 | # the sake of the other CockroachDB Pods even before they're ready, since they 39 | # have to be able to talk to each other in order to become ready. 40 | publishNotReadyAddresses: true 41 | ports: 42 | {{- $ports := .Values.service.ports }} 43 | # The main port, served by gRPC, serves Postgres-flavor SQL, inter-node 44 | # traffic and the CLI. 45 | - name: {{ $ports.grpc.external.name | quote }} 46 | port: {{ $ports.grpc.external.port | int64 }} 47 | targetPort: grpc 48 | {{- if ne ($ports.grpc.internal.port | int64) ($ports.grpc.external.port | int64) }} 49 | - name: {{ $ports.grpc.internal.name | quote }} 50 | port: {{ $ports.grpc.internal.port | int64 }} 51 | targetPort: grpc 52 | {{- end }} 53 | # The secondary port serves the UI as well as health and debug endpoints. 54 | - name: {{ $ports.http.name | quote }} 55 | port: {{ $ports.http.port | int64 }} 56 | targetPort: http 57 | selector: 58 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 59 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 60 | {{- with .Values.statefulset.labels }} 61 | {{- toYaml . | nindent 4 }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Current Operator version 2 | VERSION ?= 5.0.4 3 | # Default bundle image tag 4 | BUNDLE_IMG ?= controller-bundle:$(VERSION) 5 | # Options for 'bundle-build' 6 | ifneq ($(origin CHANNELS), undefined) 7 | BUNDLE_CHANNELS := --channels=$(CHANNELS) 8 | endif 9 | ifneq ($(origin DEFAULT_CHANNEL), undefined) 10 | BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) 11 | endif 12 | BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) 13 | 14 | # Image URL to use all building/pushing image targets 15 | IMG ?= quay.io/helmoperators/cockroachdb:v5.0.3 16 | 17 | all: docker-build 18 | 19 | # Run against the configured Kubernetes cluster in ~/.kube/config 20 | run: helm-operator 21 | $(HELM_OPERATOR) run 22 | 23 | # Install CRDs into a cluster 24 | install: kustomize 25 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 26 | 27 | # Uninstall CRDs from a cluster 28 | uninstall: kustomize 29 | $(KUSTOMIZE) build config/crd | kubectl delete -f - 30 | 31 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 32 | deploy: kustomize 33 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 34 | $(KUSTOMIZE) build config/default | kubectl apply -f - 35 | 36 | # Undeploy controller in the configured Kubernetes cluster in ~/.kube/config 37 | undeploy: kustomize 38 | $(KUSTOMIZE) build config/default | kubectl delete -f - 39 | 40 | # Build the docker image 41 | docker-build: 42 | docker build . -t ${IMG} 43 | 44 | # Push the docker image 45 | docker-push: 46 | docker push ${IMG} 47 | 48 | PATH := $(PATH):$(PWD)/bin 49 | SHELL := env PATH=$(PATH) /bin/sh 50 | OS = $(shell uname -s | tr '[:upper:]' '[:lower:]') 51 | ARCH = $(shell uname -m | sed 's/x86_64/amd64/') 52 | OSOPER = $(shell uname -s | tr '[:upper:]' '[:lower:]' | sed 's/darwin/apple-darwin/' | sed 's/linux/linux-gnu/') 53 | ARCHOPER = $(shell uname -m ) 54 | 55 | kustomize: 56 | ifeq (, $(shell which kustomize 2>/dev/null)) 57 | @{ \ 58 | set -e ;\ 59 | mkdir -p bin ;\ 60 | curl -sSLo - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.5.4/kustomize_v3.5.4_$(OS)_$(ARCH).tar.gz | tar xzf - -C bin/ ;\ 61 | } 62 | KUSTOMIZE=$(realpath ./bin/kustomize) 63 | else 64 | KUSTOMIZE=$(shell which kustomize) 65 | endif 66 | 67 | helm-operator: 68 | ifeq (, $(shell which helm-operator 2>/dev/null)) 69 | @{ \ 70 | set -e ;\ 71 | mkdir -p bin ;\ 72 | curl -LO https://github.com/operator-framework/operator-sdk/releases/download/v1.3.0/helm-operator-v1.3.0-$(ARCHOPER)-$(OSOPER) ;\ 73 | mv helm-operator-v1.3.0-$(ARCHOPER)-$(OSOPER) ./bin/helm-operator ;\ 74 | chmod +x ./bin/helm-operator ;\ 75 | } 76 | HELM_OPERATOR=$(realpath ./bin/helm-operator) 77 | else 78 | HELM_OPERATOR=$(shell which helm-operator) 79 | endif 80 | 81 | # Generate bundle manifests and metadata, then validate generated files. 82 | .PHONY: bundle 83 | bundle: kustomize 84 | operator-sdk generate kustomize manifests -q 85 | cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) 86 | $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) 87 | operator-sdk bundle validate ./bundle 88 | 89 | # Build the bundle image. 90 | .PHONY: bundle-build 91 | bundle-build: 92 | docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . 93 | -------------------------------------------------------------------------------- /config/samples/charts.operatorhub.io_v1alpha1_cockroachdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: charts.operatorhub.io/v1alpha1 2 | kind: Cockroachdb 3 | metadata: 4 | name: cockroachdb-sample 5 | spec: 6 | # Default values copied from /helm-charts/cockroachdb/values.yaml 7 | clusterDomain: cluster.local 8 | conf: 9 | attrs: [] 10 | cache: 25% 11 | cluster-name: "" 12 | disable-cluster-name-verification: false 13 | http-port: 8080 14 | join: [] 15 | locality: "" 16 | logtostderr: INFO 17 | max-sql-memory: 25% 18 | port: 26257 19 | single-node: false 20 | sql-audit-dir: "" 21 | image: 22 | credentials: {} 23 | pullPolicy: IfNotPresent 24 | repository: cockroachdb/cockroach 25 | tag: v20.2.4 26 | ingress: 27 | annotations: {} 28 | enabled: false 29 | hosts: [] 30 | labels: {} 31 | paths: 32 | - / 33 | tls: [] 34 | init: 35 | affinity: {} 36 | annotations: {} 37 | labels: 38 | app.kubernetes.io/component: init 39 | nodeSelector: {} 40 | resources: {} 41 | tolerations: [] 42 | labels: {} 43 | networkPolicy: 44 | enabled: false 45 | ingress: 46 | grpc: [] 47 | http: [] 48 | service: 49 | discovery: 50 | annotations: {} 51 | labels: 52 | app.kubernetes.io/component: cockroachdb 53 | ports: 54 | grpc: 55 | external: 56 | name: grpc 57 | port: 26257 58 | internal: 59 | name: grpc-internal 60 | port: 26257 61 | http: 62 | name: http 63 | port: 8080 64 | public: 65 | annotations: {} 66 | labels: 67 | app.kubernetes.io/component: cockroachdb 68 | type: ClusterIP 69 | serviceMonitor: 70 | annotations: {} 71 | enabled: false 72 | interval: 10s 73 | labels: {} 74 | statefulset: 75 | annotations: {} 76 | args: [] 77 | budget: 78 | maxUnavailable: 1 79 | env: [] 80 | labels: 81 | app.kubernetes.io/component: cockroachdb 82 | nodeAffinity: {} 83 | nodeSelector: {} 84 | podAffinity: {} 85 | podAntiAffinity: 86 | topologyKey: kubernetes.io/hostname 87 | type: soft 88 | weight: 100 89 | podManagementPolicy: Parallel 90 | priorityClassName: "" 91 | replicas: 3 92 | resources: {} 93 | secretMounts: [] 94 | tolerations: [] 95 | topologySpreadConstraints: 96 | maxSkew: 1 97 | topologyKey: topology.kubernetes.io/zone 98 | whenUnsatisfiable: ScheduleAnyway 99 | updateStrategy: 100 | type: RollingUpdate 101 | storage: 102 | hostPath: "" 103 | persistentVolume: 104 | annotations: {} 105 | enabled: true 106 | labels: {} 107 | size: 100Gi 108 | storageClass: "" 109 | tls: 110 | certs: 111 | clientRootSecret: cockroachdb-root 112 | nodeSecret: cockroachdb-node 113 | provided: false 114 | tlsSecret: false 115 | enabled: false 116 | init: 117 | image: 118 | credentials: {} 119 | pullPolicy: IfNotPresent 120 | repository: cockroachdb/cockroach-k8s-request-cert 121 | tag: "0.4" 122 | serviceAccount: 123 | create: true 124 | name: "" 125 | 126 | 127 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/job.init.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (eq (len .Values.conf.join) 0) (not (index .Values.conf `single-node`)) }} 2 | kind: Job 3 | apiVersion: batch/v1 4 | metadata: 5 | name: {{ template "cockroachdb.fullname" . }}-init 6 | namespace: {{ .Release.Namespace | quote }} 7 | labels: 8 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 9 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 12 | {{- with .Values.init.labels }} 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | {{- with .Values.labels }} 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | spec: 19 | template: 20 | metadata: 21 | labels: 22 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 23 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 24 | {{- with .Values.init.labels }} 25 | {{- toYaml . | nindent 8 }} 26 | {{- end }} 27 | {{- with .Values.init.annotations }} 28 | annotations: {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | spec: 31 | restartPolicy: OnFailure 32 | terminationGracePeriodSeconds: 0 33 | {{- if or .Values.image.credentials (and .Values.tls.enabled .Values.tls.init.image.credentials (not .Values.tls.certs.provided)) }} 34 | imagePullSecrets: 35 | {{- if .Values.image.credentials }} 36 | - name: {{ template "cockroachdb.fullname" . }}.db.registry 37 | {{- end }} 38 | {{- if and .Values.tls.enabled .Values.tls.init.image.credentials (not .Values.tls.certs.provided) }} 39 | - name: {{ template "cockroachdb.fullname" . }}.init-certs.registry 40 | {{- end }} 41 | {{- end }} 42 | {{- if and .Values.tls.enabled (not .Values.tls.certs.provided)}} 43 | serviceAccountName: {{ template "cockroachdb.tls.serviceAccount.name" . }} 44 | initContainers: 45 | # The init-certs container sends a CSR (certificate signing request) to 46 | # the Kubernetes cluster. 47 | # You can see pending requests using: 48 | # kubectl get csr 49 | # CSRs can be approved using: 50 | # kubectl certificate approve 51 | # 52 | # In addition to the Node certificate and key, the init-certs entrypoint 53 | # will symlink the cluster CA to the certs directory. 54 | - name: init-certs 55 | image: "{{ .Values.tls.init.image.repository }}:{{ .Values.tls.init.image.tag }}" 56 | imagePullPolicy: {{ .Values.tls.init.image.pullPolicy | quote }} 57 | command: 58 | - /bin/ash 59 | - -ecx 60 | - >- 61 | /request-cert 62 | -namespace=${POD_NAMESPACE} 63 | -certs-dir=/cockroach-certs/ 64 | -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt 65 | -type=client 66 | -user=root 67 | env: 68 | - name: POD_NAMESPACE 69 | valueFrom: 70 | fieldRef: 71 | fieldPath: metadata.namespace 72 | volumeMounts: 73 | - name: client-certs 74 | mountPath: /cockroach-certs/ 75 | {{- end }} 76 | {{- with .Values.init.affinity }} 77 | affinity: {{- toYaml . | nindent 8 }} 78 | {{- end }} 79 | {{- with .Values.init.nodeSelector }} 80 | nodeSelector: {{- toYaml . | nindent 8 }} 81 | {{- end }} 82 | {{- with .Values.init.tolerations }} 83 | tolerations: {{- toYaml . | nindent 8 }} 84 | {{- end }} 85 | containers: 86 | - name: cluster-init 87 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 88 | imagePullPolicy: {{ .Values.image.pullPolicy | quote }} 89 | # Run the command in an `while true` loop because this Job is bound 90 | # to come up before the CockroachDB Pods (due to the time needed to 91 | # get PersistentVolumes attached to Nodes), and sleeping 5 seconds 92 | # between attempts is much better than letting the Pod fail when 93 | # the init command does and waiting out Kubernetes' non-configurable 94 | # exponential back-off for Pod restarts. 95 | # Command completes either when cluster initialization succeeds, 96 | # or when cluster has been initialized already. 97 | command: 98 | - /bin/bash 99 | - -c 100 | - >- 101 | while true; do 102 | initOUT=$(set -x; 103 | /cockroach/cockroach init 104 | {{- if .Values.tls.enabled }} 105 | --certs-dir=/cockroach-certs/ 106 | {{- else }} 107 | --insecure 108 | {{- end }} 109 | {{- with index .Values.conf "cluster-name" }} 110 | --cluster-name={{.}} 111 | {{- end }} 112 | --host={{ template "cockroachdb.fullname" . }}-0.{{ template "cockroachdb.fullname" . -}} 113 | :{{ .Values.service.ports.grpc.internal.port | int64 }} 114 | 2>&1); 115 | initRC="$?"; 116 | echo $initOUT; 117 | [[ "$initRC" == "0" ]] && exit 0; 118 | [[ "$initOUT" == *"cluster has already been initialized"* ]] && exit 0; 119 | sleep 5; 120 | done 121 | {{- if .Values.tls.enabled }} 122 | volumeMounts: 123 | - name: client-certs 124 | mountPath: /cockroach-certs/ 125 | {{- end }} 126 | {{- with .Values.init.resources }} 127 | resources: {{- toYaml . | nindent 12 }} 128 | {{- end }} 129 | {{- if .Values.tls.enabled }} 130 | volumes: 131 | - name: client-certs 132 | {{- if .Values.tls.certs.provided }} 133 | {{- if .Values.tls.certs.tlsSecret }} 134 | projected: 135 | sources: 136 | - secret: 137 | name: {{ .Values.tls.certs.clientRootSecret }} 138 | items: 139 | - key: ca.crt 140 | path: ca.crt 141 | mode: 0400 142 | - key: tls.crt 143 | path: client.root.crt 144 | mode: 0400 145 | - key: tls.key 146 | path: client.root.key 147 | mode: 0400 148 | {{- else }} 149 | secret: 150 | secretName: {{ .Values.tls.certs.clientRootSecret }} 151 | defaultMode: 0400 152 | {{- end }} 153 | {{- else }} 154 | emptyDir: {} 155 | {{- end }} 156 | {{- end }} 157 | {{- end }} 158 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/templates/statefulset.yaml: -------------------------------------------------------------------------------- 1 | kind: StatefulSet 2 | apiVersion: {{ template "cockroachdb.statefulset.apiVersion" . }} 3 | metadata: 4 | name: {{ template "cockroachdb.fullname" . }} 5 | namespace: {{ .Release.Namespace | quote }} 6 | labels: 7 | helm.sh/chart: {{ template "cockroachdb.chart" . }} 8 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 9 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 10 | app.kubernetes.io/managed-by: {{ .Release.Service | quote }} 11 | {{- with .Values.statefulset.labels }} 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | {{- with .Values.labels }} 15 | {{- toYaml . | nindent 4 }} 16 | {{- end }} 17 | spec: 18 | serviceName: {{ template "cockroachdb.fullname" . }} 19 | replicas: {{ .Values.statefulset.replicas | int64 }} 20 | updateStrategy: {{- toYaml .Values.statefulset.updateStrategy | nindent 4 }} 21 | podManagementPolicy: {{ .Values.statefulset.podManagementPolicy | quote }} 22 | selector: 23 | matchLabels: 24 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 25 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 26 | {{- with .Values.statefulset.labels }} 27 | {{- toYaml . | nindent 6 }} 28 | {{- end }} 29 | template: 30 | metadata: 31 | labels: 32 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 33 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 34 | {{- with .Values.statefulset.labels }} 35 | {{- toYaml . | nindent 8 }} 36 | {{- end }} 37 | {{- with .Values.labels }} 38 | {{- toYaml . | nindent 8 }} 39 | {{- end }} 40 | {{- with .Values.statefulset.annotations }} 41 | annotations: {{- toYaml . | nindent 8 }} 42 | {{- end }} 43 | spec: 44 | {{- if or .Values.image.credentials (and .Values.tls.enabled .Values.tls.init.image.credentials (not .Values.tls.certs.provided)) }} 45 | imagePullSecrets: 46 | {{- if .Values.image.credentials }} 47 | - name: {{ template "cockroachdb.fullname" . }}.db.registry 48 | {{- end }} 49 | {{- if and .Values.tls.enabled .Values.tls.init.image.credentials (not .Values.tls.certs.provided) }} 50 | - name: {{ template "cockroachdb.fullname" . }}.init-certs.registry 51 | {{- end }} 52 | {{- end }} 53 | {{- if .Values.tls.enabled }} 54 | serviceAccountName: {{ template "cockroachdb.tls.serviceAccount.name" . }} 55 | {{- if not .Values.tls.certs.provided }} 56 | initContainers: 57 | # The init-certs container sends a CSR (certificate signing request) to 58 | # the Kubernetes cluster. 59 | # You can see pending requests using: 60 | # kubectl get csr 61 | # CSRs can be approved using: 62 | # kubectl certificate approve 63 | # 64 | # All addresses used to contact a Node must be specified in the 65 | # `--addresses` arg. 66 | # 67 | # In addition to the Node certificate and key, the init-certs entrypoint 68 | # will symlink the cluster CA to the certs directory. 69 | - name: init-certs 70 | image: "{{ .Values.tls.init.image.repository }}:{{ .Values.tls.init.image.tag }}" 71 | imagePullPolicy: {{ .Values.tls.init.image.pullPolicy | quote }} 72 | command: 73 | - /bin/ash 74 | - -ecx 75 | - >- 76 | /request-cert 77 | -namespace=${POD_NAMESPACE} 78 | -certs-dir=/cockroach-certs/ 79 | -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt 80 | -type=node 81 | -addresses=localhost,127.0.0.1,$(hostname -f),$(hostname -f|cut -f 1-2 -d '.'),{{ template "cockroachdb.fullname" . }}-public,{{ template "cockroachdb.fullname" . }}-public.$(hostname -f|cut -f 3- -d '.') 82 | env: 83 | - name: POD_NAMESPACE 84 | valueFrom: 85 | fieldRef: 86 | fieldPath: metadata.namespace 87 | volumeMounts: 88 | - name: certs 89 | mountPath: /cockroach-certs/ 90 | {{- end }} 91 | {{- end }} 92 | {{- if or .Values.statefulset.nodeAffinity .Values.statefulset.podAffinity .Values.statefulset.podAntiAffinity }} 93 | affinity: 94 | {{- with .Values.statefulset.nodeAffinity }} 95 | nodeAffinity: {{- toYaml . | nindent 10 }} 96 | {{- end }} 97 | {{- with .Values.statefulset.podAffinity }} 98 | podAffinity: {{- toYaml . | nindent 10 }} 99 | {{- end }} 100 | {{- if .Values.statefulset.podAntiAffinity }} 101 | podAntiAffinity: 102 | {{- if .Values.statefulset.podAntiAffinity.type }} 103 | {{- if eq .Values.statefulset.podAntiAffinity.type "hard" }} 104 | requiredDuringSchedulingIgnoredDuringExecution: 105 | - topologyKey: {{ .Values.statefulset.podAntiAffinity.topologyKey }} 106 | labelSelector: 107 | matchLabels: 108 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 109 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 110 | {{- with .Values.statefulset.labels }} 111 | {{- toYaml . | nindent 18 }} 112 | {{- end }} 113 | {{- else if eq .Values.statefulset.podAntiAffinity.type "soft" }} 114 | preferredDuringSchedulingIgnoredDuringExecution: 115 | - weight: {{ .Values.statefulset.podAntiAffinity.weight | int64 }} 116 | podAffinityTerm: 117 | topologyKey: {{ .Values.statefulset.podAntiAffinity.topologyKey }} 118 | labelSelector: 119 | matchLabels: 120 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 121 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 122 | {{- with .Values.statefulset.labels }} 123 | {{- toYaml . | nindent 20 }} 124 | {{- end }} 125 | {{- end }} 126 | {{- else }} 127 | {{- toYaml .Values.statefulset.podAntiAffinity | nindent 10 }} 128 | {{- end }} 129 | {{- end }} 130 | {{- end }} 131 | {{- if semverCompare ">=1.16-0" .Capabilities.KubeVersion.GitVersion }} 132 | topologySpreadConstraints: 133 | - labelSelector: 134 | matchLabels: 135 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 136 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 137 | {{- with .Values.statefulset.labels }} 138 | {{- toYaml . | nindent 12 }} 139 | {{- end }} 140 | {{- with .Values.statefulset.topologySpreadConstraints }} 141 | maxSkew: {{ .maxSkew }} 142 | topologyKey: {{ .topologyKey }} 143 | whenUnsatisfiable: {{ .whenUnsatisfiable }} 144 | {{- end }} 145 | {{- end }} 146 | {{- with .Values.statefulset.nodeSelector }} 147 | nodeSelector: {{- toYaml . | nindent 8 }} 148 | {{- end }} 149 | {{- if .Values.statefulset.priorityClassName }} 150 | priorityClassName: {{ .Values.statefulset.priorityClassName }} 151 | {{- end }} 152 | {{- with .Values.statefulset.tolerations }} 153 | tolerations: {{- toYaml . | nindent 8 }} 154 | {{- end }} 155 | # No pre-stop hook is required, a SIGTERM plus some time is all that's 156 | # needed for graceful shutdown of a node. 157 | terminationGracePeriodSeconds: 60 158 | containers: 159 | - name: db 160 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 161 | imagePullPolicy: {{ .Values.image.pullPolicy | quote }} 162 | args: 163 | - shell 164 | - -ecx 165 | # The use of qualified `hostname -f` is crucial: 166 | # Other nodes aren't able to look up the unqualified hostname. 167 | # 168 | # `--join` CLI flag is hardcoded to exactly 3 Pods, because: 169 | # 1. Having `--join` value depending on `statefulset.replicas` 170 | # will trigger undesired restart of existing Pods when 171 | # StatefulSet is scaled up/down. We want to scale without 172 | # restarting existing Pods. 173 | # 2. At least one Pod in `--join` is enough to successfully 174 | # join CockroachDB cluster and gossip with all other existing 175 | # Pods, even if there are 3 or more Pods. 176 | # 3. It's harmless for `--join` to have 3 Pods even for 1-Pod 177 | # clusters, while it gives us opportunity to scale up even if 178 | # some Pods of existing cluster are down (for whatever reason). 179 | # See details explained here: 180 | # https://github.com/helm/charts/pull/18993#issuecomment-558795102 181 | - >- 182 | exec /cockroach/cockroach 183 | {{- if index .Values.conf `single-node` }} 184 | start-single-node 185 | {{- else }} 186 | start --join= 187 | {{- if .Values.conf.join }} 188 | {{- join `,` .Values.conf.join -}} 189 | {{- else }} 190 | {{- range $i, $_ := until 3 -}} 191 | {{- if gt $i 0 -}},{{- end -}} 192 | ${STATEFULSET_NAME}-{{ $i }}.${STATEFULSET_FQDN}:{{ $.Values.service.ports.grpc.internal.port | int64 -}} 193 | {{- end -}} 194 | {{- end }} 195 | --advertise-host=$(hostname).${STATEFULSET_FQDN} 196 | {{- with index .Values.conf `cluster-name` }} 197 | --cluster-name={{ . }} 198 | {{- if index $.Values.conf `disable-cluster-name-verification` }} 199 | --disable-cluster-name-verification 200 | {{- end }} 201 | {{- end }} 202 | {{- end }} 203 | --logtostderr={{ .Values.conf.logtostderr }} 204 | {{- if .Values.tls.enabled }} 205 | --certs-dir=/cockroach/cockroach-certs/ 206 | {{- else }} 207 | --insecure 208 | {{- end }} 209 | {{- with .Values.conf.attrs }} 210 | --attrs={{ join `:` . }} 211 | {{- end }} 212 | --http-port={{ index .Values.conf `http-port` | int64 }} 213 | --port={{ .Values.conf.port | int64 }} 214 | --cache={{ .Values.conf.cache }} 215 | {{- with index .Values.conf `max-disk-temp-storage` }} 216 | --max-disk-temp-storage={{ . }} 217 | {{- end }} 218 | {{- with index .Values.conf `max-offset` }} 219 | --max-offset={{ . }} 220 | {{- end }} 221 | --max-sql-memory={{ index .Values.conf `max-sql-memory` }} 222 | {{- with .Values.conf.locality }} 223 | --locality={{ . }} 224 | {{- end }} 225 | {{- with index .Values.conf `sql-audit-dir` }} 226 | --sql-audit-dir={{ . }} 227 | {{- end }} 228 | {{- range .Values.statefulset.args }} 229 | {{ . }} 230 | {{- end }} 231 | env: 232 | - name: STATEFULSET_NAME 233 | value: {{ template "cockroachdb.fullname" . }} 234 | - name: STATEFULSET_FQDN 235 | value: {{ template "cockroachdb.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} 236 | - name: COCKROACH_CHANNEL 237 | value: kubernetes-helm 238 | {{- with .Values.statefulset.env }} 239 | {{- toYaml . | nindent 12 }} 240 | {{- end }} 241 | ports: 242 | - name: grpc 243 | containerPort: {{ .Values.conf.port | int64 }} 244 | protocol: TCP 245 | - name: http 246 | containerPort: {{ index .Values.conf `http-port` | int64 }} 247 | protocol: TCP 248 | volumeMounts: 249 | - name: datadir 250 | mountPath: /cockroach/cockroach-data/ 251 | {{- if .Values.tls.enabled }} 252 | - name: certs 253 | mountPath: /cockroach/cockroach-certs/ 254 | {{- end }} 255 | {{- range .Values.statefulset.secretMounts }} 256 | - name: {{ printf "secret-%s" . | quote }} 257 | mountPath: {{ printf "/etc/cockroach/secrets/%s" . | quote }} 258 | readOnly: true 259 | {{- end }} 260 | livenessProbe: 261 | httpGet: 262 | path: /health 263 | port: http 264 | {{- if .Values.tls.enabled }} 265 | scheme: HTTPS 266 | {{- end }} 267 | initialDelaySeconds: 30 268 | periodSeconds: 5 269 | readinessProbe: 270 | httpGet: 271 | path: /health?ready=1 272 | port: http 273 | {{- if .Values.tls.enabled }} 274 | scheme: HTTPS 275 | {{- end }} 276 | initialDelaySeconds: 10 277 | periodSeconds: 5 278 | failureThreshold: 2 279 | {{- with .Values.statefulset.resources }} 280 | resources: {{- toYaml . | nindent 12 }} 281 | {{- end }} 282 | volumes: 283 | - name: datadir 284 | {{- if .Values.storage.persistentVolume.enabled }} 285 | persistentVolumeClaim: 286 | claimName: datadir 287 | {{- else if .Values.storage.hostPath }} 288 | hostPath: 289 | path: {{ .Values.storage.hostPath | quote }} 290 | {{- else }} 291 | emptyDir: {} 292 | {{- end }} 293 | {{- if .Values.tls.enabled }} 294 | - name: certs 295 | {{- if .Values.tls.certs.provided }} 296 | {{- if .Values.tls.certs.tlsSecret }} 297 | projected: 298 | sources: 299 | - secret: 300 | name: {{ .Values.tls.certs.nodeSecret }} 301 | items: 302 | - key: ca.crt 303 | path: ca.crt 304 | mode: 0400 305 | - key: tls.crt 306 | path: node.crt 307 | mode: 0400 308 | - key: tls.key 309 | path: node.key 310 | mode: 0400 311 | {{- else }} 312 | secret: 313 | secretName: {{ .Values.tls.certs.nodeSecret }} 314 | defaultMode: 0400 315 | {{- end }} 316 | {{- else }} 317 | emptyDir: {} 318 | {{- end }} 319 | {{- end }} 320 | {{- range .Values.statefulset.secretMounts }} 321 | - name: {{ printf "secret-%s" . | quote }} 322 | secret: 323 | secretName: {{ . | quote }} 324 | {{- end }} 325 | {{- if .Values.storage.persistentVolume.enabled }} 326 | volumeClaimTemplates: 327 | - metadata: 328 | name: datadir 329 | labels: 330 | app.kubernetes.io/name: {{ template "cockroachdb.name" . }} 331 | app.kubernetes.io/instance: {{ .Release.Name | quote }} 332 | {{- with .Values.storage.persistentVolume.labels }} 333 | {{- toYaml . | nindent 10 }} 334 | {{- end }} 335 | {{- with .Values.labels }} 336 | {{- toYaml . | nindent 10 }} 337 | {{- end }} 338 | {{- with .Values.storage.persistentVolume.annotations }} 339 | annotations: {{- toYaml . | nindent 10 }} 340 | {{- end }} 341 | spec: 342 | accessModes: ["ReadWriteOnce"] 343 | {{- if .Values.storage.persistentVolume.storageClass }} 344 | {{- if (eq "-" .Values.storage.persistentVolume.storageClass) }} 345 | storageClassName: "" 346 | {{- else }} 347 | storageClassName: {{ .Values.storage.persistentVolume.storageClass | quote}} 348 | {{- end }} 349 | {{- end }} 350 | resources: 351 | requests: 352 | storage: {{ .Values.storage.persistentVolume.size | quote }} 353 | {{- end }} 354 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: cockroachdb/cockroach 3 | tag: v20.2.4 4 | pullPolicy: IfNotPresent 5 | credentials: {} 6 | # registry: docker.io 7 | # username: john_doe 8 | # password: changeme 9 | 10 | 11 | # Additional labels to apply to all Kubernetes resources created by this chart. 12 | labels: {} 13 | # app.kubernetes.io/part-of: my-app 14 | 15 | 16 | # Cluster's default DNS domain. 17 | # You should overwrite it if you're using a different one, 18 | # otherwise CockroachDB nodes discovery won't work. 19 | clusterDomain: cluster.local 20 | 21 | 22 | conf: 23 | # An ordered list of CockroachDB node attributes. 24 | # Attributes are arbitrary strings specifying machine capabilities. 25 | # Machine capabilities might include specialized hardware or number of cores 26 | # (e.g. "gpu", "x16c"). 27 | attrs: [] 28 | # - x16c 29 | # - gpu 30 | 31 | # Total size in bytes for caches, shared evenly if there are multiple 32 | # storage devices. Size suffixes are supported (e.g. `1GB` and `1GiB`). 33 | # A percentage of physical memory can also be specified (e.g. `.25`). 34 | cache: 25% 35 | 36 | # Sets a name to verify the identity of a cluster. 37 | # The value must match between all nodes specified via `conf.join`. 38 | # This can be used as an additional verification when either the node or 39 | # cluster, or both, have not yet been initialized and do not yet know their 40 | # cluster ID. 41 | # To introduce a cluster name into an already-initialized cluster, pair this 42 | # option with `conf.disable-cluster-name-verification: yes`. 43 | cluster-name: "" 44 | 45 | # Tell the server to ignore `conf.cluster-name` mismatches. 46 | # This is meant for use when opting an existing cluster into starting to use 47 | # cluster name verification, or when changing the cluster name. 48 | # The cluster should be restarted once with `conf.cluster-name` and 49 | # `conf.disable-cluster-name-verification: yes` combined, and once all nodes 50 | # have been updated to know the new cluster name, the cluster can be restarted 51 | # again with `conf.disable-cluster-name-verification: no`. 52 | # This option has no effect if `conf.cluster-name` is not specified. 53 | disable-cluster-name-verification: false 54 | 55 | # The addresses for connecting a CockroachDB nodes to an existing cluster. 56 | # If you are deploying a second CockroachDB instance that should join a first 57 | # one, use the below list to join to the existing instance. 58 | # Each item in the array should be a FQDN (and port if needed) resolvable by 59 | # new Pods. 60 | join: [] 61 | 62 | # Logs at or above this threshold to STDERR. 63 | logtostderr: INFO 64 | 65 | # Maximum storage capacity available to store temporary disk-based data for 66 | # SQL queries that exceed the memory budget (e.g. join, sorts, etc are 67 | # sometimes able to spill intermediate results to disk). 68 | # Accepts numbers interpreted as bytes, size suffixes (e.g. `32GB` and 69 | # `32GiB`) or a percentage of disk size (e.g. `10%`). 70 | # The location of the temporary files is within the first store dir. 71 | # If expressed as a percentage, `max-disk-temp-storage` is interpreted 72 | # relative to the size of the storage device on which the first store is 73 | # placed. The temp space usage is never counted towards any store usage 74 | # (although it does share the device with the first store) so, when 75 | # configuring this, make sure that the size of this temp storage plus the size 76 | # of the first store don't exceed the capacity of the storage device. 77 | # If the first store is an in-memory one (i.e. `type=mem`), then this 78 | # temporary "disk" data is also kept in-memory. 79 | # A percentage value is interpreted as a percentage of the available internal 80 | # memory. 81 | # max-disk-temp-storage: 0GB 82 | 83 | # Maximum allowed clock offset for the cluster. If observed clock offsets 84 | # exceed this limit, servers will crash to minimize the likelihood of 85 | # reading inconsistent data. Increasing this value will increase the time 86 | # to recovery of failures as well as the frequency of uncertainty-based 87 | # read restarts. 88 | # Note, that this value must be the same on all nodes in the cluster. 89 | # In order to change it, all nodes in the cluster must be stopped 90 | # simultaneously and restarted with the new value. 91 | # max-offset: 500ms 92 | 93 | # Maximum memory capacity available to store temporary data for SQL clients, 94 | # including prepared queries and intermediate data rows during query 95 | # execution. Accepts numbers interpreted as bytes, size suffixes 96 | # (e.g. `1GB` and `1GiB`) or a percentage of physical memory (e.g. `.25`). 97 | max-sql-memory: 25% 98 | 99 | # An ordered, comma-separated list of key-value pairs that describe the 100 | # topography of the machine. Topography might include country, datacenter 101 | # or rack designations. Data is automatically replicated to maximize 102 | # diversities of each tier. The order of tiers is used to determine 103 | # the priority of the diversity, so the more inclusive localities like 104 | # country should come before less inclusive localities like datacenter. 105 | # The tiers and order must be the same on all nodes. Including more tiers 106 | # is better than including fewer. For example: 107 | # locality: country=us,region=us-west,datacenter=us-west-1b,rack=12 108 | # locality: country=ca,region=ca-east,datacenter=ca-east-2,rack=4 109 | # locality: planet=earth,province=manitoba,colo=secondary,power=3 110 | locality: "" 111 | 112 | # Run CockroachDB instances in standalone mode with replication disabled 113 | # (replication factor = 1). 114 | # Enabling this option makes the following values to be ignored: 115 | # - `conf.cluster-name` 116 | # - `conf.disable-cluster-name-verification` 117 | # - `conf.join` 118 | # 119 | # WARNING: Enabling this option makes each deployed Pod as a STANDALONE 120 | # CockroachDB instance, so the StatefulSet does NOT FORM A CLUSTER. 121 | # Don't use this option for production deployments unless you clearly 122 | # understand what you're doing. 123 | # Usually, this option is intended to be used in conjunction with 124 | # `statefulset.replicas: 1` for temporary one-time deployments (like 125 | # running E2E tests, for example). 126 | single-node: false 127 | 128 | # If non-empty, create a SQL audit log in the specified directory. 129 | sql-audit-dir: "" 130 | 131 | # CockroachDB's port to listen to inter-communications and client connections. 132 | port: 26257 133 | 134 | # CockroachDB's port to listen to HTTP requests. 135 | http-port: 8080 136 | 137 | 138 | statefulset: 139 | replicas: 3 140 | updateStrategy: 141 | type: RollingUpdate 142 | podManagementPolicy: Parallel 143 | budget: 144 | maxUnavailable: 1 145 | 146 | # List of additional command-line arguments you want to pass to the 147 | # `cockroach start` command. 148 | args: [] 149 | # - --disable-cluster-name-verification 150 | 151 | # List of extra environment variables to pass into container 152 | env: [] 153 | # - name: COCKROACH_ENGINE_MAX_SYNC_DURATION 154 | # value: "24h" 155 | 156 | # List of Secrets names in the same Namespace as the CockroachDB cluster, 157 | # which shall be mounted into `/etc/cockroach/secrets/` for every cluster 158 | # member. 159 | secretMounts: [] 160 | 161 | # Additional labels to apply to this StatefulSet and all its Pods. 162 | labels: 163 | app.kubernetes.io/component: cockroachdb 164 | 165 | # Additional annotations to apply to the Pods of this StatefulSet. 166 | annotations: {} 167 | 168 | # Affinity rules for scheduling Pods of this StatefulSet on Nodes. 169 | # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity 170 | nodeAffinity: {} 171 | # Inter-Pod Affinity rules for scheduling Pods of this StatefulSet. 172 | # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity 173 | podAffinity: {} 174 | # Anti-affinity rules for scheduling Pods of this StatefulSet. 175 | # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity 176 | # You may either toggle options below for default anti-affinity rules, 177 | # or specify the whole set of anti-affinity rules instead of them. 178 | podAntiAffinity: 179 | # The topologyKey to be used. 180 | # Can be used to spread across different nodes, AZs, regions etc. 181 | topologyKey: kubernetes.io/hostname 182 | # Type of anti-affinity rules: either `soft`, `hard` or empty value (which 183 | # disables anti-affinity rules). 184 | type: soft 185 | # Weight for `soft` anti-affinity rules. 186 | # Does not apply for other anti-affinity types. 187 | weight: 100 188 | 189 | # Node selection constraints for scheduling Pods of this StatefulSet. 190 | # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector 191 | nodeSelector: {} 192 | 193 | # PriorityClassName given to Pods of this StatefulSet 194 | # https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass 195 | priorityClassName: "" 196 | 197 | # Taints to be tolerated by Pods of this StatefulSet. 198 | # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ 199 | tolerations: [] 200 | 201 | # https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ 202 | topologySpreadConstraints: 203 | maxSkew: 1 204 | topologyKey: topology.kubernetes.io/zone 205 | whenUnsatisfiable: ScheduleAnyway 206 | 207 | # Uncomment the following resources definitions or pass them from 208 | # command line to control the CPU and memory resources allocated 209 | # by Pods of this StatefulSet. 210 | resources: {} 211 | # limits: 212 | # cpu: 100m 213 | # memory: 512Mi 214 | # requests: 215 | # cpu: 100m 216 | # memory: 512Mi 217 | 218 | service: 219 | ports: 220 | # You can set a different external and internal gRPC ports and their name. 221 | grpc: 222 | external: 223 | port: 26257 224 | name: grpc 225 | # If the port number is different than `external.port`, then it will be 226 | # named as `internal.name` in Service. 227 | internal: 228 | port: 26257 229 | # If using Istio set it to `cockroach`. 230 | name: grpc-internal 231 | http: 232 | port: 8080 233 | name: http 234 | 235 | # This Service is meant to be used by clients of the database. 236 | # It exposes a ClusterIP that will automatically load balance connections 237 | # to the different database Pods. 238 | public: 239 | type: ClusterIP 240 | # Additional labels to apply to this Service. 241 | labels: 242 | app.kubernetes.io/component: cockroachdb 243 | # Additional annotations to apply to this Service. 244 | annotations: {} 245 | 246 | # This service only exists to create DNS entries for each pod in 247 | # the StatefulSet such that they can resolve each other's IP addresses. 248 | # It does not create a load-balanced ClusterIP and should not be used directly 249 | # by clients in most circumstances. 250 | discovery: 251 | # Additional labels to apply to this Service. 252 | labels: 253 | app.kubernetes.io/component: cockroachdb 254 | # Additional annotations to apply to this Service. 255 | annotations: {} 256 | 257 | # CockroachDB's ingress for web ui. 258 | ingress: 259 | enabled: false 260 | labels: {} 261 | annotations: {} 262 | # kubernetes.io/ingress.class: nginx 263 | # cert-manager.io/cluster-issuer: letsencrypt 264 | paths: [/] 265 | hosts: [] 266 | # - cockroachlabs.com 267 | tls: [] 268 | # - hosts: [cockroachlabs.com] 269 | # secretName: cockroachlabs-tls 270 | 271 | # CockroachDB's Prometheus operator ServiceMonitor support 272 | serviceMonitor: 273 | enabled: false 274 | labels: {} 275 | annotations: {} 276 | interval: 10s 277 | # scrapeTimeout: 10s 278 | 279 | # CockroachDB's data persistence. 280 | # If neither `persistentVolume` nor `hostPath` is used, then data will be 281 | # persisted in ad-hoc `emptyDir`. 282 | storage: 283 | # Absolute path on host to store CockroachDB's data. 284 | # If not specified, then `emptyDir` will be used instead. 285 | # If specified, but `persistentVolume.enabled` is `true`, then has no effect. 286 | hostPath: "" 287 | 288 | # If `enabled` is `true` then a PersistentVolumeClaim will be created and 289 | # used to store CockroachDB's data, otherwise `hostPath` is used. 290 | persistentVolume: 291 | enabled: true 292 | 293 | size: 100Gi 294 | 295 | # If defined, then `storageClassName: `. 296 | # If set to "-", then `storageClassName: ""`, which disables dynamic 297 | # provisioning. 298 | # If undefined or empty (default), then no `storageClassName` spec is set, 299 | # so the default provisioner will be chosen (gp2 on AWS, standard on 300 | # GKE, AWS & OpenStack). 301 | storageClass: "" 302 | 303 | # Additional labels to apply to the created PersistentVolumeClaims. 304 | labels: {} 305 | # Additional annotations to apply to the created PersistentVolumeClaims. 306 | annotations: {} 307 | 308 | 309 | # Kubernetes Job which initializes multi-node CockroachDB cluster. 310 | # It's not created if `statefulset.replicas` is `1`. 311 | init: 312 | # Additional labels to apply to this Job and its Pod. 313 | labels: 314 | app.kubernetes.io/component: init 315 | 316 | # Additional annotations to apply to the Pod of this Job. 317 | annotations: {} 318 | 319 | # Affinity rules for scheduling the Pod of this Job. 320 | # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity 321 | affinity: {} 322 | 323 | # Node selection constraints for scheduling the Pod of this Job. 324 | # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector 325 | nodeSelector: {} 326 | 327 | # Taints to be tolerated by the Pod of this Job. 328 | # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ 329 | tolerations: [] 330 | 331 | # The init Pod runs at cluster creation to initialize CockroachDB. It finishes 332 | # quickly and doesn't continue to consume resources in the Kubernetes 333 | # cluster. Normally, you should leave this section commented out, but if your 334 | # Kubernetes cluster uses Resource Quotas and requires all pods to specify 335 | # resource requests or limits, you can set those here. 336 | resources: {} 337 | # requests: 338 | # cpu: "10m" 339 | # memory: "128Mi" 340 | # limits: 341 | # cpu: "10m" 342 | # memory: "128Mi" 343 | 344 | 345 | # Whether to run securely using TLS certificates. 346 | tls: 347 | enabled: false 348 | serviceAccount: 349 | # Specifies whether this ServiceAccount should be created. 350 | create: true 351 | # The name of this ServiceAccount to use. 352 | # If not set and `create` is `true`, then a name is auto-generated. 353 | name: "" 354 | certs: 355 | # Bring your own certs scenario. If provided, tls.init section will be ignored. 356 | provided: false 357 | # Secret name for the client root cert. 358 | clientRootSecret: cockroachdb-root 359 | # Secret name for node cert. 360 | nodeSecret: cockroachdb-node 361 | # Enable if the secret is a dedicated TLS. 362 | # TLS secrets are created by cert-mananger, for example. 363 | tlsSecret: false 364 | 365 | init: 366 | # Image to use for requesting TLS certificates. 367 | image: 368 | repository: cockroachdb/cockroach-k8s-request-cert 369 | tag: "0.4" 370 | pullPolicy: IfNotPresent 371 | credentials: {} 372 | # registry: docker.io 373 | # username: john_doe 374 | # password: changeme 375 | 376 | 377 | networkPolicy: 378 | enabled: false 379 | 380 | ingress: 381 | # List of sources which should be able to access the CockroachDB Pods via 382 | # gRPC port. Items in this list are combined using a logical OR operation. 383 | # Rules for allowing inter-communication are applied automatically. 384 | # If empty, then connections from any Pod is allowed. 385 | grpc: [] 386 | # - podSelector: 387 | # matchLabels: 388 | # app.kubernetes.io/name: my-app-django 389 | # app.kubernetes.io/instance: my-app 390 | 391 | # List of sources which should be able to access the CockroachDB Pods via 392 | # HTTP port. Items in this list are combined using a logical OR operation. 393 | # If empty, then connections from any Pod is allowed. 394 | http: [] 395 | # - namespaceSelector: 396 | # matchLabels: 397 | # project: my-project 398 | -------------------------------------------------------------------------------- /helm-charts/cockroachdb/README.md: -------------------------------------------------------------------------------- 1 | # CockroachDB Helm Chart 2 | 3 | [CockroachDB](https://github.com/cockroachdb/cockroach) - the open source, cloud-native distributed SQL database. 4 | 5 | ## Documentation 6 | 7 | Below is a brief overview of operating the CockroachDB Helm Chart and some specific implementation details. For additional information on deploying CockroachDB, please see: 8 | > 9 | 10 | Note that the documentation requires Helm 3.0 or higher. 11 | 12 | ## Prerequisites Details 13 | 14 | * Kubernetes 1.8 15 | * PV support on the underlying infrastructure (only if using `storage.persistentVolume`). [Docker for windows hostpath provisioner is not supported](https://github.com/cockroachdb/docs/issues/3184). 16 | * If you want to secure your cluster to use TLS certificates for all network communication, [Helm must be installed with RBAC privileges](https://helm.sh/docs/topics/rbac/) or else you will get an "attempt to grant extra privileges" error. 17 | 18 | ## StatefulSet Details 19 | 20 | * 21 | 22 | ## StatefulSet Caveats 23 | 24 | * 25 | 26 | ## Chart Details 27 | 28 | This chart will do the following: 29 | 30 | * Set up a dynamically scalable CockroachDB cluster using a Kubernetes StatefulSet. 31 | 32 | ## Add the CockroachDB Repository 33 | 34 | ```shell 35 | helm repo add cockroachdb https://charts.cockroachdb.com/ 36 | ``` 37 | 38 | ## Installing the Chart 39 | 40 | To install the chart with the release name `my-release`: 41 | 42 | ```shell 43 | helm install my-release cockroachdb/cockroachdb 44 | ``` 45 | 46 | Note that for a production cluster, you will likely want to override the following parameters in [`values.yaml`](values.yaml) with your own values. 47 | 48 | - `statefulset.resources.requests.memory` and `statefulset.resources.limits.memory` allocate memory resources to CockroachDB pods in your cluster. 49 | - `conf.cache` and `conf.max-sql-memory` are memory limits that we recommend setting to 1/4 of the above resource allocation. When running CockroachDB, you must set these limits explicitly to avoid running out of memory. 50 | - `storage.persistentVolume.size` defaults to `100Gi` of disk space per pod, which you may increase or decrease for your use case. 51 | - `storage.persistentVolume.storageClass` uses the default storage class for your environment. We strongly recommend that you specify a storage class which uses an SSD. 52 | - `tls.enabled` must be set to `yes`/`true` to deploy in secure mode. 53 | 54 | For more information on overriding the `values.yaml` parameters, please see: 55 | > 56 | 57 | If you are running in secure mode (with configuration parameter `tls.enabled` set to `yes`/`true`) and `tls.certs.provided` set to `no`/`false`), then you will have to manually approve the cluster's security certificates as the pods are created. You can see the pending CSRs (certificate signing requests) by running `kubectl get csr`, and approve them by running `kubectl certificate approve `. You'll have to approve one certificate for each CockroachDB node (e.g., `default.node.my-release-cockroachdb-0` and one client certificate for the job that initializes the cluster (e.g., `default.node.root`). 58 | 59 | When `tls.certs.provided` is set to `yes`/`true`, this chart will use certificates created outside of Kubernetes. You may want to use this if you want to use a different certificate authority from the one being used by Kubernetes or if your Kubernetes cluster doesn't fully support certificate-signing requests. To use this, first set up your certificates and load them into your Kubernetes cluster as Secrets using the commands below: 60 | 61 | ``` 62 | mkdir certs 63 | mkdir my-safe-directory 64 | cockroach cert create-ca --certs-dir=certs --ca-key=my-safe-directory/ca.key 65 | cockroach cert create-client root --certs-dir=certs --ca-key=my-safe-directory/ca.key 66 | kubectl create secret generic cockroachdb-root --from-file=certs 67 | cockroach cert create-node --certs-dir=certs --ca-key=my-safe-directory/ca.key localhost 127.0.0.1 eerie-horse-cockroachdb-public eerie-horse-cockroachdb-public.default eerie-horse-cockroachdb-public.default.svc.cluster.local *.eerie-horse-cockroachdb *.eerie-horse-cockroachdb.default *.eerie-horse-cockroachdb.default.svc.cluster.local 68 | kubectl create secret generic cockroachdb-node --from-file=certs 69 | ``` 70 | 71 | Set `tls.certs.tlsSecret` to `yes/true` if you make use of [cert-manager][3] in your cluster. 72 | 73 | [cert-manager][3] stores generated certificates in dedicated TLS secrets. Thus, they are always named: 74 | 75 | * `ca.crt` 76 | * `tls.crt` 77 | * `tls.key` 78 | 79 | On the other hand, CockroachDB also demands dedicated certificate filenames: 80 | 81 | * `ca.crt` 82 | * `node.crt` 83 | * `node.key` 84 | * `client.root.crt` 85 | * `client.root.key` 86 | 87 | By activating `tls.certs.tlsSecret` we benefit from projected secrets and convert the TLS secret filenames to their according CockroachDB filenames. 88 | 89 | If you are running in secure mode, then you will have to manually approve the cluster's security certificates as the pods are created. You can see the pending CSRs (certificate signing requests) by running `kubectl get csr`, and approve them by running `kubectl certificate approve `. You'll have to approve one certificate for each CockroachDB node (e.g., `default.node.my-release-cockroachdb-0` and one client certificate for the job that initializes the cluster (e.g., `default.node.root`). 90 | 91 | Confirm that all pods are `Running` successfully and init has been completed: 92 | 93 | ```shell 94 | kubectl get pods 95 | ``` 96 | 97 | ``` 98 | NAME READY STATUS RESTARTS AGE 99 | my-release-cockroachdb-0 1/1 Running 0 1m 100 | my-release-cockroachdb-1 1/1 Running 0 1m 101 | my-release-cockroachdb-2 1/1 Running 0 1m 102 | my-release-cockroachdb-init-k6jcr 0/1 Completed 0 1m 103 | ``` 104 | 105 | Confirm that persistent volumes are created and claimed for each pod: 106 | 107 | ```shell 108 | kubectl get pv 109 | ``` 110 | 111 | ``` 112 | NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE 113 | pvc-64878ebf-f3f0-11e8-ab5b-42010a8e0035 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-0 standard 51s 114 | pvc-64945b4f-f3f0-11e8-ab5b-42010a8e0035 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-1 standard 51s 115 | pvc-649d920d-f3f0-11e8-ab5b-42010a8e0035 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-2 standard 51s 116 | ``` 117 | 118 | ## Upgrading the cluster 119 | 120 | ### Chart version 3.0.0 and after 121 | 122 | Launch a temporary interactive pod and start the built-in SQL client: 123 | 124 | ```shell 125 | kubectl run cockroachdb --rm -it \ 126 | --image=cockroachdb/cockroach \ 127 | --restart=Never \ 128 | -- sql --insecure --host=my-release-cockroachdb-public 129 | ``` 130 | 131 | > If you are running in secure mode, you will have to provide a client certificate to the cluster in order to authenticate, so the above command will not work. See [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) for an example of how to set up an interactive SQL shell against a secure cluster or [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/example-app-secure.yaml) for an example application connecting to a secure cluster. 132 | 133 | Set `cluster.preserve_downgrade_option`, where `$current_version` is the CockroachDB version currently running (e.g., `19.2`): 134 | 135 | ```sql 136 | > SET CLUSTER SETTING cluster.preserve_downgrade_option = '$current_version'; 137 | ``` 138 | 139 | Exit the shell and delete the temporary pod: 140 | 141 | ```sql 142 | > \q 143 | ``` 144 | 145 | Kick off the upgrade process by changing the new Docker image, where `$new_version` is the CockroachDB version to which you are upgrading: 146 | 147 | ```shell 148 | kubectl delete job my-release-cockroachdb-init 149 | ``` 150 | 151 | ```shell 152 | helm upgrade my-release cockroachdb/cockroachdb \ 153 | --set image.tag=$new_version \ 154 | --reuse-values 155 | ``` 156 | 157 | Kubernetes will carry out a safe [rolling upgrade](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) of your CockroachDB nodes one-by-one. Monitor the cluster's pods until all have been successfully restarted: 158 | 159 | ```shell 160 | kubectl get pods 161 | ``` 162 | 163 | ``` 164 | NAME READY STATUS RESTARTS AGE 165 | my-release-cockroachdb-0 1/1 Running 0 2m 166 | my-release-cockroachdb-1 1/1 Running 0 3m 167 | my-release-cockroachdb-2 1/1 Running 0 3m 168 | my-release-cockroachdb-3 0/1 ContainerCreating 0 25s 169 | my-release-cockroachdb-init-nwjkh 0/1 ContainerCreating 0 6s 170 | ``` 171 | 172 | ```shell 173 | kubectl get pods \ 174 | -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[0].image}{"\n"}' 175 | ``` 176 | 177 | ``` 178 | my-release-cockroachdb-0 cockroachdb/cockroach:v20.2.4 179 | my-release-cockroachdb-1 cockroachdb/cockroach:v20.2.4 180 | my-release-cockroachdb-2 cockroachdb/cockroach:v20.2.4 181 | my-release-cockroachdb-3 cockroachdb/cockroach:v20.2.4 182 | ``` 183 | 184 | Resume normal operations. Once you are comfortable that the stability and performance of the cluster is what you'd expect post-upgrade, finalize the upgrade: 185 | 186 | ```shell 187 | kubectl run cockroachdb --rm -it \ 188 | --image=cockroachdb/cockroach \ 189 | --restart=Never \ 190 | -- sql --insecure --host=my-release-cockroachdb-public 191 | ``` 192 | 193 | ```sql 194 | > RESET CLUSTER SETTING cluster.preserve_downgrade_option; 195 | > \q 196 | ``` 197 | 198 | ### Chart versions prior to 3.0.0 199 | 200 | Due to a change in the label format in version 3.0.0 of this chart, upgrading requires that you delete the StatefulSet. Luckily there is a way to do it without actually deleting all the resources managed by the StatefulSet. Use the workaround below to upgrade from charts versions previous to 3.0.0: 201 | 202 | Get the new labels from the specs rendered by Helm: 203 | 204 | ```shell 205 | helm template -f deploy.vals.yml cockroachdb/cockroachdb -x templates/statefulset.yaml \ 206 | | yq r - spec.template.metadata.labels 207 | ``` 208 | 209 | ``` 210 | app.kubernetes.io/name: cockroachdb 211 | app.kubernetes.io/instance: my-release 212 | app.kubernetes.io/component: cockroachdb 213 | ``` 214 | 215 | Place the new labels on all pods of the StatefulSet (change `my-release-cockroachdb-0` to the name of each pod): 216 | 217 | ```shell 218 | kubectl label pods my-release-cockroachdb-0 \ 219 | app.kubernetes.io/name=cockroachdb \ 220 | app.kubernetes.io/instance=my-release \ 221 | app.kubernetes.io/component=cockroachdb 222 | ``` 223 | 224 | Delete the StatefulSet without deleting pods: 225 | 226 | ```shell 227 | kubectl delete statefulset my-release-cockroachdb --cascade=false 228 | ``` 229 | 230 | Verify that no pod is deleted and then upgrade as normal. A new StatefulSet will be created, taking over the management of the existing pods and upgrading them if needed. 231 | 232 | ### See also 233 | 234 | For more information about upgrading a cluster to the latest major release of CockroachDB, see [Upgrade to CockroachDB v20.2](https://www.cockroachlabs.com/docs/stable/upgrade-cockroach-version.html). 235 | 236 | Note that there are some backward-incompatible changes to SQL features between versions 20.1 and 20.2. For details, see the [CockroachDB v20.2.0 release notes](https://www.cockroachlabs.com/docs/releases/v20.2.0.html#backward-incompatible-changes). 237 | 238 | ## Configuration 239 | 240 | The following table lists the configurable parameters of the CockroachDB chart and their default values. 241 | For details see the [`values.yaml`](values.yaml) file. 242 | 243 | | Parameter | Description | Default | 244 | | --------- | ----------- | ------- | 245 | | `clusterDomain` | Cluster's default DNS domain | `cluster.local` | 246 | | `conf.attrs` | CockroachDB node attributes | `[]` | 247 | | `conf.cache` | Size of CockroachDB's in-memory cache | `25%` | 248 | | `conf.cluster-name` | Name of CockroachDB cluster | `""` | 249 | | `conf.disable-cluster-name-verification` | Disable CockroachDB cluster name verification | `no` | 250 | | `conf.join` | List of already-existing CockroachDB instances | `[]` | 251 | | `conf.max-disk-temp-storage` | Max storage capacity for temp data | `0` | 252 | | `conf.max-offset` | Max allowed clock offset for CockroachDB cluster | `500ms` | 253 | | `conf.max-sql-memory` | Max memory to use processing SQL querie | `25%` | 254 | | `conf.locality` | Locality attribute for this deployment | `""` | 255 | | `conf.single-node` | Disable CockroachDB clustering (standalone mode) | `no` | 256 | | `conf.sql-audit-dir` | Directory for SQL audit log | `""` | 257 | | `conf.port` | CockroachDB primary serving port in Pods | `26257` | 258 | | `conf.http-port` | CockroachDB HTTP port in Pods | `8080` | 259 | | `image.repository` | Container image name | `cockroachdb/cockroach` | 260 | | `image.tag` | Container image tag | `v20.2.4` | 261 | | `image.pullPolicy` | Container pull policy | `IfNotPresent` | 262 | | `image.credentials` | `registry`, `user` and `pass` credentials to pull private image | `{}` | 263 | | `statefulset.replicas` | StatefulSet replicas number | `3` | 264 | | `statefulset.updateStrategy` | Update strategy for StatefulSet Pods | `{"type": "RollingUpdate"}` | 265 | | `statefulset.podManagementPolicy` | `OrderedReady`/`Parallel` Pods creation/deletion order | `Parallel` | 266 | | `statefulset.budget.maxUnavailable` | k8s PodDisruptionBudget parameter | `1` | 267 | | `statefulset.args` | Extra command-line arguments | `[]` | 268 | | `statefulset.env` | Extra env vars | `[]` | 269 | | `statefulset.secretMounts` | Additional Secrets to mount at cluster members | `[]` | 270 | | `statefulset.labels` | Additional labels of StatefulSet and its Pods | `{"app.kubernetes.io/component": "cockroachdb"}` | 271 | | `statefulset.annotations` | Additional annotations of StatefulSet Pods | `{}` | 272 | | `statefulset.nodeAffinity` | [Node affinity rules][2] of StatefulSet Pods | `{}` | 273 | | `statefulset.podAffinity` | [Inter-Pod affinity rules][1] of StatefulSet Pods | `{}` | 274 | | `statefulset.podAntiAffinity` | [Anti-affinity rules][1] of StatefulSet Pods | auto | 275 | | `statefulset.podAntiAffinity.topologyKey` | The topologyKey for auto [anti-affinity rules][1] | `kubernetes.io/hostname` | 276 | | `statefulset.podAntiAffinity.type` | Type of auto [anti-affinity rules][1] | `soft` | 277 | | `statefulset.podAntiAffinity.weight` | Weight for `soft` auto [anti-affinity rules][1] | `100` | 278 | | `statefulset.nodeSelector` | Node labels for StatefulSet Pods assignment | `{}` | 279 | | `statefulset.priorityClassName` | [PriorityClassName][4] for StatefulSet Pods | `""` | 280 | | `statefulset.tolerations` | Node taints to tolerate by StatefulSet Pods | `[]` | 281 | | `statefulset.topologySpreadConstraints` | [Topology Spread Constraints rules][5] of StatefulSet Pods | auto | 282 | | `statefulset.topologySpreadConstraints.maxSkew` | Degree to which Pods may be unevenly distributed | `1` | 283 | | `statefulset.topologySpreadConstraints.topologyKey` | The key of node labels | `topology.kubernetes.io/zone` | 284 | | `statefulset.topologySpreadConstraints.whenUnsatisfiable` | `ScheduleAnyway`/`DoNotSchedule` for unsatisfiable constraints | `ScheduleAnyway` | 285 | | `statefulset.resources` | Resource requests and limits for StatefulSet Pods | `{}` | 286 | | `service.ports.grpc.external.port` | CockroachDB primary serving port in Services | `26257` | 287 | | `service.ports.grpc.external.name` | CockroachDB primary serving port name in Services | `grpc` | 288 | | `service.ports.grpc.internal.port` | CockroachDB inter-communication port in Services | `26257` | 289 | | `service.ports.grpc.internal.name` | CockroachDB inter-communication port name in Services | `grpc-internal` | 290 | | `service.ports.http.port` | CockroachDB HTTP port in Services | `8080` | 291 | | `service.ports.http.name` | CockroachDB HTTP port name in Services | `http` | 292 | | `service.public.type` | Public Service type | `ClusterIP` | 293 | | `service.public.labels` | Additional labels of public Service | `{"app.kubernetes.io/component": "cockroachdb"}` | 294 | | `service.public.annotations` | Additional annotations of public Service | `{}` | 295 | | `service.discovery.labels` | Additional labels of discovery Service | `{"app.kubernetes.io/component": "cockroachdb"}` | 296 | | `service.discovery.annotations` | Additional annotations of discovery Service | `{}` | 297 | | `ingress.enabled` | Enable ingress resource for CockroachDB | `false` | 298 | | `ingress.labels` | Additional labels of Ingress | `{}` | 299 | | `ingress.annotations` | Additional annotations of Ingress | `{}` | 300 | | `ingress.paths` | Paths for the default host | `[/]` | 301 | | `ingress.hosts` | CockroachDB Ingress hostnames | `[]` | 302 | | `ingress.tls[0].hosts` | CockroachDB Ingress tls hostnames | `nil` | 303 | | `ingress.tls[0].secretName` | CockroachDB Ingress tls secret name | `nil` | 304 | | `serviceMonitor.enabled` | Create [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md#servicemonitor) Resource for scraping metrics using [PrometheusOperator](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md#prometheus-operator) | `false` | 305 | | `serviceMonitor.labels` | Additional labels of ServiceMonitor | `{}` | 306 | | `serviceMonitor.annotations` | Additional annotations of ServiceMonitor | `{}` | 307 | | `serviceMonitor.interval` | ServiceMonitor scrape metrics interval | `10s` | 308 | | `serviceMonitor.scrapeTimeout` | ServiceMonitor scrape timeout | `nil` | 309 | | `storage.hostPath` | Absolute path on host to store data | `""` | 310 | | `storage.persistentVolume.enabled` | Whether to use PersistentVolume to store data | `yes` | 311 | | `storage.persistentVolume.size` | PersistentVolume size | `100Gi` | 312 | | `storage.persistentVolume.storageClass` | PersistentVolume class | `""` | 313 | | `storage.persistentVolume.labels` | Additional labels of PersistentVolumeClaim | `{}` | 314 | | `storage.persistentVolume.annotations` | Additional annotations of PersistentVolumeClaim | `{}` | 315 | | `init.labels` | Additional labels of init Job and its Pod | `{"app.kubernetes.io/component": "init"}` | 316 | | `init.annotations` | Additional labels of the Pod of init Job | `{}` | 317 | | `init.affinity` | [Affinity rules][2] of init Job Pod | `{}` | 318 | | `init.nodeSelector` | Node labels for init Job Pod assignment | `{}` | 319 | | `init.tolerations` | Node taints to tolerate by init Job Pod | `[]` | 320 | | `init.resources` | Resource requests and limits for the Pod of init Job | `{}` | 321 | | `tls.enabled` | Whether to run securely using TLS certificates | `no` | 322 | | `tls.serviceAccount.create` | Whether to create a new RBAC service account | `yes` | 323 | | `tls.serviceAccount.name` | Name of RBAC service account to use | `""` | 324 | | `tls.certs.provided` | Bring your own certs scenario, i.e certificates are provided | `no` | 325 | | `tls.certs.clientRootSecret` | If certs are provided, secret name for client root cert | `cockroachdb-root` | 326 | | `tls.certs.nodeSecret` | If certs are provided, secret name for node cert | `cockroachdb-node` | 327 | | `tls.certs.tlsSecret` | Own certs are stored in TLS secret | `no` | 328 | | `tls.init.image.repository` | Image to use for requesting TLS certificates | `cockroachdb/cockroach-k8s-request-cert` | 329 | | `tls.init.image.tag` | Image tag to use for requesting TLS certificates | `0.4` | 330 | | `tls.init.image.pullPolicy` | Requesting TLS certificates container pull policy | `IfNotPresent` | 331 | | `tls.init.image.credentials` | `registry`, `user` and `pass` credentials to pull private image | `{}` | 332 | | `networkPolicy.enabled` | Enable NetworkPolicy for CockroachDB's Pods | `no` | 333 | | `networkPolicy.ingress.grpc` | Whitelist resources to access gRPC port of CockroachDB's Pods | `[]` | 334 | | `networkPolicy.ingress.http` | Whitelist resources to access gRPC port of CockroachDB's Pods | `[]` | 335 | 336 | 337 | Override the default parameters using the `--set key=value[,key=value]` argument to `helm install`. 338 | 339 | Alternatively, a YAML file that specifies custom values for the parameters can be provided while installing the chart. For example: 340 | 341 | ```shell 342 | helm install my-release -f my-values.yaml cockroachdb/cockroachdb 343 | ``` 344 | 345 | > **Tip**: You can use the default [values.yaml](values.yaml) 346 | 347 | ## Deep dive 348 | 349 | ### Connecting to the CockroachDB cluster 350 | 351 | Once you've created the cluster, you can start talking to it by connecting to its `-public` Service. CockroachDB is PostgreSQL wire protocol compatible, so there's a [wide variety of supported clients](https://www.cockroachlabs.com/docs/install-client-drivers.html). As an example, we'll open up a SQL shell using CockroachDB's built-in shell and play around with it a bit, like this (likely needing to replace `my-release-cockroachdb-public` with the name of the `-public` Service that was created with your installed chart): 352 | 353 | ```shell 354 | kubectl run cockroach-client --rm -it \ 355 | --image=cockroachdb/cockroach \ 356 | --restart=Never \ 357 | -- sql --insecure --host my-release-cockroachdb-public 358 | ``` 359 | 360 | ``` 361 | Waiting for pod default/cockroach-client to be running, status is Pending, 362 | pod ready: false 363 | If you don't see a command prompt, try pressing enter. 364 | root@my-release-cockroachdb-public:26257> SHOW DATABASES; 365 | +--------------------+ 366 | | Database | 367 | +--------------------+ 368 | | information_schema | 369 | | pg_catalog | 370 | | system | 371 | +--------------------+ 372 | (3 rows) 373 | root@my-release-cockroachdb-public:26257> CREATE DATABASE bank; 374 | CREATE DATABASE 375 | root@my-release-cockroachdb-public:26257> CREATE TABLE bank.accounts (id INT 376 | PRIMARY KEY, balance DECIMAL); 377 | CREATE TABLE 378 | root@my-release-cockroachdb-public:26257> INSERT INTO bank.accounts VALUES 379 | (1234, 10000.50); 380 | INSERT 1 381 | root@my-release-cockroachdb-public:26257> SELECT * FROM bank.accounts; 382 | +------+---------+ 383 | | id | balance | 384 | +------+---------+ 385 | | 1234 | 10000.5 | 386 | +------+---------+ 387 | (1 row) 388 | root@my-release-cockroachdb-public:26257> \q 389 | Waiting for pod default/cockroach-client to terminate, status is Running 390 | pod "cockroach-client" deleted 391 | ``` 392 | 393 | > If you are running in secure mode, you will have to provide a client certificate to the cluster in order to authenticate, so the above command will not work. See [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) for an example of how to set up an interactive SQL shell against a secure cluster or [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/example-app-secure.yaml) for an example application connecting to a secure cluster. 394 | 395 | ### Cluster health 396 | 397 | Because our pod spec includes regular health checks of the CockroachDB processes, simply running `kubectl get pods` and looking at the `STATUS` column is sufficient to determine the health of each instance in the cluster. 398 | 399 | If you want more detailed information about the cluster, the best place to look is the Admin UI. 400 | 401 | ### Accessing the Admin UI 402 | 403 | If you want to see information about how the cluster is doing, you can try pulling up the CockroachDB Admin UI by port-forwarding from your local machine to one of the pods (replacing `my-release-cockroachdb-0` with the name of one of your pods: 404 | 405 | ```shell 406 | kubectl port-forward my-release-cockroachdb-0 8080 407 | ``` 408 | 409 | You should then be able to access the Admin UI by visiting in your web browser. 410 | 411 | ### Failover 412 | 413 | If any CockroachDB member fails, it is restarted or recreated automatically by the Kubernetes infrastructure, and will re-join the cluster automatically when it comes back up. You can test this scenario by killing any of the CockroachDB pods: 414 | 415 | ```shell 416 | kubectl delete pod my-release-cockroachdb-1 417 | ``` 418 | 419 | ```shell 420 | kubectl get pods -l "app.kubernetes.io/instance=my-release,app.kubernetes.io/component=cockroachdb" 421 | ``` 422 | 423 | ``` 424 | NAME READY STATUS RESTARTS AGE 425 | my-release-cockroachdb-0 1/1 Running 0 5m 426 | my-release-cockroachdb-2 1/1 Running 0 5m 427 | ``` 428 | 429 | After a while: 430 | 431 | ```shell 432 | kubectl get pods -l "app.kubernetes.io/instance=my-release,app.kubernetes.io/component=cockroachdb" 433 | ``` 434 | 435 | ``` 436 | NAME READY STATUS RESTARTS AGE 437 | my-release-cockroachdb-0 1/1 Running 0 5m 438 | my-release-cockroachdb-1 1/1 Running 0 20s 439 | my-release-cockroachdb-2 1/1 Running 0 5m 440 | ``` 441 | 442 | You can check the state of re-joining from the new pod's logs: 443 | 444 | ```shell 445 | kubectl logs my-release-cockroachdb-1 446 | ``` 447 | 448 | ``` 449 | [...] 450 | I161028 19:32:09.754026 1 server/node.go:586 [n1] node connected via gossip and 451 | verified as part of cluster {"35ecbc27-3f67-4e7d-9b8f-27c31aae17d6"} 452 | [...] 453 | cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local:26257 454 | build: beta-20161027-55-gd2d3c7f @ 2016/10/28 19:27:25 (go1.7.3) 455 | admin: http://0.0.0.0:8080 456 | sql: 457 | postgresql://root@my-release-cockroachdb-1.my-release-cockroachdb.default.svc.cluster.local:26257?sslmode=disable 458 | logs: cockroach-data/logs 459 | store[0]: path=cockroach-data 460 | status: restarted pre-existing node 461 | clusterID: {35ecbc27-3f67-4e7d-9b8f-27c31aae17d6} 462 | nodeID: 2 463 | [...] 464 | ``` 465 | 466 | ### NetworkPolicy 467 | 468 | To enable NetworkPolicy for CockroachDB, install [a networking plugin that implements the Kubernetes NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), and set `networkPolicy.enabled` to `yes`/`true`. 469 | 470 | For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting the `DefaultDeny` Namespace annotation. Note: this will enforce policy for _all_ pods in the Namespace: 471 | 472 | ```shell 473 | kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" 474 | ``` 475 | 476 | For more precise policy, set `networkPolicy.ingress.grpc` and `networkPolicy.ingress.http` rules. This will only allow pods that match the provided rules to connect to CockroachDB. 477 | 478 | ### Scaling 479 | 480 | Scaling should be managed via the `helm upgrade` command. After resizing your cluster on your cloud environment (e.g., GKE or EKS), run the following command to add a pod. This assumes you scaled from 3 to 4 nodes: 481 | 482 | ```shell 483 | helm upgrade \ 484 | my-release \ 485 | cockroachdb/cockroachdb \ 486 | --set statefulset.replicas=4 \ 487 | --reuse-values 488 | ``` 489 | 490 | Note, that if you are running in secure mode (`tls.enabled` is `yes`/`true`) and increase the size of your cluster, you will also have to approve the CSR (certificate-signing request) of each new node (using `kubectl get csr` and `kubectl certificate approve`). 491 | 492 | [1]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity 493 | [2]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity 494 | [3]: https://cert-manager.io/ 495 | [4]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass 496 | [5]: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ 497 | -------------------------------------------------------------------------------- /config/manifests/bases/cockroachdb.clusterserviceversion.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1alpha1 2 | kind: ClusterServiceVersion 3 | metadata: 4 | annotations: 5 | alm-examples: '[]' 6 | capabilities: Basic Install 7 | categories: Database 8 | certified: "false" 9 | containerImage: quay.io/helmoperators/cockroachdb:v5.0.4 10 | createdAt: 2021-01-23T10-00-00Z 11 | description: CockroachDB Operator based on the CockroachDB helm chart 12 | repository: https://github.com/dmesser/cockroachdb-operator 13 | support: dmesser 14 | name: cockroachdb.v5.0.4 15 | namespace: placeholder 16 | spec: 17 | apiservicedefinitions: {} 18 | customresourcedefinitions: 19 | owned: 20 | - description: Represents a CockroachDB cluster 21 | displayName: CockroachDB 22 | kind: Cockroachdb 23 | name: cockroachdbs.charts.operatorhub.io 24 | resources: 25 | - kind: Deployment 26 | name: "" 27 | version: apps/v1 28 | - kind: Service 29 | name: "" 30 | version: v1 31 | - kind: ReplicaSet 32 | name: "" 33 | version: apps/v1 34 | - kind: Pod 35 | name: "" 36 | version: v1 37 | - kind: Secret 38 | name: "" 39 | version: v1 40 | - kind: ConfigMap 41 | name: "" 42 | version: v1 43 | - kind: PersistentVolumeClaim 44 | name: "" 45 | version: v1 46 | - kind: StatefulSet 47 | name: "" 48 | version: apps/v1 49 | - kind: Job 50 | name: "" 51 | version: batch/v1 52 | version: v1alpha1 53 | description: | 54 | CockroachDB is a scalable, survivable, strongly-consistent SQL database. 55 | ## About this Operator 56 | This Operator is based on a Helm chart for CockroachDB. It supports reconfiguration for some parameters, but notably does not handle scale down of the replica count in a seamless manner. Scale up works great. 57 | 58 | ## Core capabilities 59 | * **StatefulSet** - Sets up a dynamically scalable CockroachDB cluster using a Kubernetes StatefulSet 60 | * **Expand Replicas** - Supports expanding the set of replicas by simply editing your object 61 | * **Dashboard** - Installs the CockroachDB user interface to administer your cluster. Easily expose it via an Ingress rule. 62 | Review all of the [confiuguration options](https://github.com/helm/charts/tree/master/stable/cockroachdb#configuration) to best run your database instance. The example configuration is derived from the chart's [`values.yaml`](https://github.com/helm/charts/blob/master/stable/cockroachdb/values.yaml). 63 | ## Using the cluster 64 | The resulting cluster endpoint can be consumed from a `Service` that follows the pattern: `-public`. For example to connect using the command line client, use something like the following to obtain the name of the service: 65 | ``` 66 | kubectl get service -l chart=cockroachdb-5.0.4 67 | NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE 68 | example-9f8ngwzrxbxrulxqmdqfhn51h-cdb ClusterIP None 26257/TCP,8080/TCP 24m 69 | example-9f8ngwzrxbxrulxqmdqfhn51h-cdb-public ClusterIP 10.106.249.134 26257/TCP,8080/TCP 24m 70 | ``` 71 | Then you can use the CockroachDB command line client to connect to the database cluster: 72 | ``` 73 | kubectl run -it --rm cockroach-client --image=cockroachdb/cockroach --restart=Never --command -- ./cockroach sql --insecure --host example-9f8ngwzrxbxrulxqmdqfhn51h-cdb-public 74 | ``` 75 | ## Before you start 76 | This Operator requires a cluster with PV support in order to run correctly. 77 | 78 | Note that for a production cluster, you will likely want to override the following parameters in [`values.yaml`](values.yaml) with your own values. 79 | 80 | - `statefulset.resources.requests.memory` and `statefulset.resources.limits.memory` allocate memory resources to CockroachDB pods in your cluster. 81 | - `conf.cache` and `conf.max-sql-memory` are memory limits that we recommend setting to 1/4 of the above resource allocation. When running CockroachDB, you must set these limits explicitly to avoid running out of memory. 82 | - `storage.persistentVolume.size` defaults to `100Gi` of disk space per pod, which you may increase or decrease for your use case. 83 | - `storage.persistentVolume.storageClass` uses the default storage class for your environment. We strongly recommend that you specify a storage class which uses an SSD. 84 | - `tls.enabled` must be set to `yes`/`true` to deploy in secure mode. 85 | 86 | For more information on overriding the `values.yaml` parameters, please see: 87 | > 88 | 89 | If you are running in secure mode (with configuration parameter `tls.enabled` set to `yes`/`true`) and `tls.certs.provided` set to `no`/`false`), then you will have to manually approve the cluster's security certificates as the pods are created. You can see the pending CSRs (certificate signing requests) by running `kubectl get csr`, and approve them by running `kubectl certificate approve `. You'll have to approve one certificate for each CockroachDB node (e.g., `default.node.my-release-cockroachdb-0` and one client certificate for the job that initializes the cluster (e.g., `default.node.root`). 90 | 91 | When `tls.certs.provided` is set to `yes`/`true`, this chart will use certificates created outside of Kubernetes. You may want to use this if you want to use a different certificate authority from the one being used by Kubernetes or if your Kubernetes cluster doesn't fully support certificate-signing requests. To use this, first set up your certificates and load them into your Kubernetes cluster as Secrets using the commands below: 92 | 93 | ``` 94 | mkdir certs 95 | mkdir my-safe-directory 96 | cockroach cert create-ca --certs-dir=certs --ca-key=my-safe-directory/ca.key 97 | cockroach cert create-client root --certs-dir=certs --ca-key=my-safe-directory/ca.key 98 | kubectl create secret generic cockroachdb-root --from-file=certs 99 | cockroach cert create-node --certs-dir=certs --ca-key=my-safe-directory/ca.key localhost 127.0.0.1 eerie-horse-cockroachdb-public eerie-horse-cockroachdb-public.default eerie-horse-cockroachdb-public.default.svc.cluster.local *.eerie-horse-cockroachdb *.eerie-horse-cockroachdb.default *.eerie-horse-cockroachdb.default.svc.cluster.local 100 | kubectl create secret generic cockroachdb-node --from-file=certs 101 | ``` 102 | 103 | Set `tls.certs.tlsSecret` to `yes/true` if you make use of [cert-manager][3] in your cluster. 104 | 105 | [cert-manager][3] stores generated certificates in dedicated TLS secrets. Thus, they are always named: 106 | 107 | * `ca.crt` 108 | * `tls.crt` 109 | * `tls.key` 110 | 111 | On the other hand, CockroachDB also demands dedicated certificate filenames: 112 | 113 | * `ca.crt` 114 | * `node.crt` 115 | * `node.key` 116 | * `client.root.crt` 117 | * `client.root.key` 118 | 119 | By activating `tls.certs.tlsSecret` we benefit from projected secrets and convert the TLS secret filenames to their according CockroachDB filenames. 120 | 121 | If you are running in secure mode, then you will have to manually approve the cluster's security certificates as the pods are created. You can see the pending CSRs (certificate signing requests) by running `kubectl get csr`, and approve them by running `kubectl certificate approve `. You'll have to approve one certificate for each CockroachDB node (e.g., `default.node.my-release-cockroachdb-0` and one client certificate for the job that initializes the cluster (e.g., `default.node.root`). 122 | 123 | Confirm that all pods are `Running` successfully and init has been completed: 124 | 125 | ```shell 126 | kubectl get pods 127 | ``` 128 | 129 | ``` 130 | NAME READY STATUS RESTARTS AGE 131 | my-release-cockroachdb-0 1/1 Running 0 1m 132 | my-release-cockroachdb-1 1/1 Running 0 1m 133 | my-release-cockroachdb-2 1/1 Running 0 1m 134 | my-release-cockroachdb-init-k6jcr 0/1 Completed 0 1m 135 | ``` 136 | 137 | Confirm that persistent volumes are created and claimed for each pod: 138 | 139 | ```shell 140 | kubectl get pv 141 | ``` 142 | 143 | ``` 144 | NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE 145 | pvc-64878ebf-f3f0-11e8-ab5b-42010a8e0035 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-0 standard 51s 146 | pvc-64945b4f-f3f0-11e8-ab5b-42010a8e0035 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-1 standard 51s 147 | pvc-649d920d-f3f0-11e8-ab5b-42010a8e0035 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-2 standard 51s 148 | ``` 149 | 150 | ## Upgrading the cluster 151 | 152 | ### Chart version 3.0.0 and after 153 | 154 | Launch a temporary interactive pod and start the built-in SQL client: 155 | 156 | ```shell 157 | kubectl run cockroachdb --rm -it \ 158 | --image=cockroachdb/cockroach \ 159 | --restart=Never \ 160 | -- sql --insecure --host=my-release-cockroachdb-public 161 | ``` 162 | 163 | > If you are running in secure mode, you will have to provide a client certificate to the cluster in order to authenticate, so the above command will not work. See [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) for an example of how to set up an interactive SQL shell against a secure cluster or [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/example-app-secure.yaml) for an example application connecting to a secure cluster. 164 | 165 | Set `cluster.preserve_downgrade_option`, where `$current_version` is the CockroachDB version currently running (e.g., `19.2`): 166 | 167 | ```sql 168 | > SET CLUSTER SETTING cluster.preserve_downgrade_option = '$current_version'; 169 | ``` 170 | 171 | Exit the shell and delete the temporary pod: 172 | 173 | ```sql 174 | > \q 175 | ``` 176 | 177 | Kick off the upgrade process by changing the new Docker image, where `$new_version` is the CockroachDB version to which you are upgrading: 178 | 179 | ```shell 180 | kubectl delete job my-release-cockroachdb-init 181 | ``` 182 | 183 | ```shell 184 | kubectl patch cockroachdbs example -p '{"spec": {"image":{"tag":""}}}' --type merge 185 | ``` 186 | 187 | Kubernetes will carry out a safe [rolling upgrade](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) of your CockroachDB nodes one-by-one. Monitor the cluster's pods until all have been successfully restarted: 188 | 189 | ```shell 190 | kubectl get pods 191 | ``` 192 | 193 | ``` 194 | NAME READY STATUS RESTARTS AGE 195 | my-release-cockroachdb-0 1/1 Running 0 2m 196 | my-release-cockroachdb-1 1/1 Running 0 3m 197 | my-release-cockroachdb-2 1/1 Running 0 3m 198 | my-release-cockroachdb-3 0/1 ContainerCreating 0 25s 199 | my-release-cockroachdb-init-nwjkh 0/1 ContainerCreating 0 6s 200 | ``` 201 | 202 | ```shell 203 | kubectl get pods \ 204 | -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[0].image}{"\n"}' 205 | ``` 206 | 207 | ``` 208 | my-release-cockroachdb-0 cockroachdb/cockroach:v20.2.4 209 | my-release-cockroachdb-1 cockroachdb/cockroach:v20.2.4 210 | my-release-cockroachdb-2 cockroachdb/cockroach:v20.2.4 211 | my-release-cockroachdb-3 cockroachdb/cockroach:v20.2.4 212 | ``` 213 | 214 | Resume normal operations. Once you are comfortable that the stability and performance of the cluster is what you'd expect post-upgrade, finalize the upgrade: 215 | 216 | ```shell 217 | kubectl run cockroachdb --rm -it \ 218 | --image=cockroachdb/cockroach \ 219 | --restart=Never \ 220 | -- sql --insecure --host=my-release-cockroachdb-public 221 | ``` 222 | 223 | ```sql 224 | > RESET CLUSTER SETTING cluster.preserve_downgrade_option; 225 | > \q 226 | ``` 227 | 228 | ## Configuration 229 | 230 | The following table lists the configurable parameters of the CockroachDB chart and their default values. 231 | For details see the [`values.yaml`](values.yaml) file. 232 | 233 | | Parameter | Description | Default | 234 | | --------- | ----------- | ------- | 235 | | `clusterDomain` | Cluster's default DNS domain | `cluster.local` | 236 | | `conf.attrs` | CockroachDB node attributes | `[]` | 237 | | `conf.cache` | Size of CockroachDB's in-memory cache | `25%` | 238 | | `conf.cluster-name` | Name of CockroachDB cluster | `""` | 239 | | `conf.disable-cluster-name-verification` | Disable CockroachDB cluster name verification | `no` | 240 | | `conf.join` | List of already-existing CockroachDB instances | `[]` | 241 | | `conf.max-disk-temp-storage` | Max storage capacity for temp data | `0` | 242 | | `conf.max-offset` | Max allowed clock offset for CockroachDB cluster | `500ms` | 243 | | `conf.max-sql-memory` | Max memory to use processing SQL querie | `25%` | 244 | | `conf.locality` | Locality attribute for this deployment | `""` | 245 | | `conf.single-node` | Disable CockroachDB clustering (standalone mode) | `no` | 246 | | `conf.sql-audit-dir` | Directory for SQL audit log | `""` | 247 | | `conf.port` | CockroachDB primary serving port in Pods | `26257` | 248 | | `conf.http-port` | CockroachDB HTTP port in Pods | `8080` | 249 | | `image.repository` | Container image name | `cockroachdb/cockroach` | 250 | | `image.tag` | Container image tag | `v20.2.4` | 251 | | `image.pullPolicy` | Container pull policy | `IfNotPresent` | 252 | | `image.credentials` | `registry`, `user` and `pass` credentials to pull private image | `{}` | 253 | | `statefulset.replicas` | StatefulSet replicas number | `3` | 254 | | `statefulset.updateStrategy` | Update strategy for StatefulSet Pods | `{"type": "RollingUpdate"}` | 255 | | `statefulset.podManagementPolicy` | `OrderedReady`/`Parallel` Pods creation/deletion order | `Parallel` | 256 | | `statefulset.budget.maxUnavailable` | k8s PodDisruptionBudget parameter | `1` | 257 | | `statefulset.args` | Extra command-line arguments | `[]` | 258 | | `statefulset.env` | Extra env vars | `[]` | 259 | | `statefulset.secretMounts` | Additional Secrets to mount at cluster members | `[]` | 260 | | `statefulset.labels` | Additional labels of StatefulSet and its Pods | `{"app.kubernetes.io/component": "cockroachdb"}` | 261 | | `statefulset.annotations` | Additional annotations of StatefulSet Pods | `{}` | 262 | | `statefulset.nodeAffinity` | [Node affinity rules][2] of StatefulSet Pods | `{}` | 263 | | `statefulset.podAffinity` | [Inter-Pod affinity rules][1] of StatefulSet Pods | `{}` | 264 | | `statefulset.podAntiAffinity` | [Anti-affinity rules][1] of StatefulSet Pods | auto | 265 | | `statefulset.podAntiAffinity.topologyKey` | The topologyKey for auto [anti-affinity rules][1] | `kubernetes.io/hostname` | 266 | | `statefulset.podAntiAffinity.type` | Type of auto [anti-affinity rules][1] | `soft` | 267 | | `statefulset.podAntiAffinity.weight` | Weight for `soft` auto [anti-affinity rules][1] | `100` | 268 | | `statefulset.nodeSelector` | Node labels for StatefulSet Pods assignment | `{}` | 269 | | `statefulset.priorityClassName` | [PriorityClassName][4] for StatefulSet Pods | `""` | 270 | | `statefulset.tolerations` | Node taints to tolerate by StatefulSet Pods | `[]` | 271 | | `statefulset.topologySpreadConstraints` | [Topology Spread Constraints rules][5] of StatefulSet Pods | auto | 272 | | `statefulset.topologySpreadConstraints.maxSkew` | Degree to which Pods may be unevenly distributed | `1` | 273 | | `statefulset.topologySpreadConstraints.topologyKey` | The key of node labels | `topology.kubernetes.io/zone` | 274 | | `statefulset.topologySpreadConstraints.whenUnsatisfiable` | `ScheduleAnyway`/`DoNotSchedule` for unsatisfiable constraints | `ScheduleAnyway` | 275 | | `statefulset.resources` | Resource requests and limits for StatefulSet Pods | `{}` | 276 | | `service.ports.grpc.external.port` | CockroachDB primary serving port in Services | `26257` | 277 | | `service.ports.grpc.external.name` | CockroachDB primary serving port name in Services | `grpc` | 278 | | `service.ports.grpc.internal.port` | CockroachDB inter-communication port in Services | `26257` | 279 | | `service.ports.grpc.internal.name` | CockroachDB inter-communication port name in Services | `grpc-internal` | 280 | | `service.ports.http.port` | CockroachDB HTTP port in Services | `8080` | 281 | | `service.ports.http.name` | CockroachDB HTTP port name in Services | `http` | 282 | | `service.public.type` | Public Service type | `ClusterIP` | 283 | | `service.public.labels` | Additional labels of public Service | `{"app.kubernetes.io/component": "cockroachdb"}` | 284 | | `service.public.annotations` | Additional annotations of public Service | `{}` | 285 | | `service.discovery.labels` | Additional labels of discovery Service | `{"app.kubernetes.io/component": "cockroachdb"}` | 286 | | `service.discovery.annotations` | Additional annotations of discovery Service | `{}` | 287 | | `ingress.enabled` | Enable ingress resource for CockroachDB | `false` | 288 | | `ingress.labels` | Additional labels of Ingress | `{}` | 289 | | `ingress.annotations` | Additional annotations of Ingress | `{}` | 290 | | `ingress.paths` | Paths for the default host | `[/]` | 291 | | `ingress.hosts` | CockroachDB Ingress hostnames | `[]` | 292 | | `ingress.tls[0].hosts` | CockroachDB Ingress tls hostnames | `nil` | 293 | | `ingress.tls[0].secretName` | CockroachDB Ingress tls secret name | `nil` | 294 | | `serviceMonitor.enabled` | Create [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md#servicemonitor) Resource for scraping metrics using [PrometheusOperator](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md#prometheus-operator) | `false` | 295 | | `serviceMonitor.labels` | Additional labels of ServiceMonitor | `{}` | 296 | | `serviceMonitor.annotations` | Additional annotations of ServiceMonitor | `{}` | 297 | | `serviceMonitor.interval` | ServiceMonitor scrape metrics interval | `10s` | 298 | | `serviceMonitor.scrapeTimeout` | ServiceMonitor scrape timeout | `nil` | 299 | | `storage.hostPath` | Absolute path on host to store data | `""` | 300 | | `storage.persistentVolume.enabled` | Whether to use PersistentVolume to store data | `yes` | 301 | | `storage.persistentVolume.size` | PersistentVolume size | `100Gi` | 302 | | `storage.persistentVolume.storageClass` | PersistentVolume class | `""` | 303 | | `storage.persistentVolume.labels` | Additional labels of PersistentVolumeClaim | `{}` | 304 | | `storage.persistentVolume.annotations` | Additional annotations of PersistentVolumeClaim | `{}` | 305 | | `init.labels` | Additional labels of init Job and its Pod | `{"app.kubernetes.io/component": "init"}` | 306 | | `init.annotations` | Additional labels of the Pod of init Job | `{}` | 307 | | `init.affinity` | [Affinity rules][2] of init Job Pod | `{}` | 308 | | `init.nodeSelector` | Node labels for init Job Pod assignment | `{}` | 309 | | `init.tolerations` | Node taints to tolerate by init Job Pod | `[]` | 310 | | `init.resources` | Resource requests and limits for the Pod of init Job | `{}` | 311 | | `tls.enabled` | Whether to run securely using TLS certificates | `no` | 312 | | `tls.serviceAccount.create` | Whether to create a new RBAC service account | `yes` | 313 | | `tls.serviceAccount.name` | Name of RBAC service account to use | `""` | 314 | | `tls.certs.provided` | Bring your own certs scenario, i.e certificates are provided | `no` | 315 | | `tls.certs.clientRootSecret` | If certs are provided, secret name for client root cert | `cockroachdb-root` | 316 | | `tls.certs.nodeSecret` | If certs are provided, secret name for node cert | `cockroachdb-node` | 317 | | `tls.certs.tlsSecret` | Own certs are stored in TLS secret | `no` | 318 | | `tls.init.image.repository` | Image to use for requesting TLS certificates | `cockroachdb/cockroach-k8s-request-cert` | 319 | | `tls.init.image.tag` | Image tag to use for requesting TLS certificates | `0.4` | 320 | | `tls.init.image.pullPolicy` | Requesting TLS certificates container pull policy | `IfNotPresent` | 321 | | `tls.init.image.credentials` | `registry`, `user` and `pass` credentials to pull private image | `{}` | 322 | | `networkPolicy.enabled` | Enable NetworkPolicy for CockroachDB's Pods | `no` | 323 | | `networkPolicy.ingress.grpc` | Whitelist resources to access gRPC port of CockroachDB's Pods | `[]` | 324 | | `networkPolicy.ingress.http` | Whitelist resources to access gRPC port of CockroachDB's Pods | `[]` | 325 | 326 | 327 | Override the default parameters using the `CockroachDB` custom resource. 328 | 329 | ## Deep dive 330 | 331 | ### Connecting to the CockroachDB cluster 332 | 333 | Once you've created the cluster, you can start talking to it by connecting to its `-public` Service. CockroachDB is PostgreSQL wire protocol compatible, so there's a [wide variety of supported clients](https://www.cockroachlabs.com/docs/install-client-drivers.html). As an example, we'll open up a SQL shell using CockroachDB's built-in shell and play around with it a bit, like this (likely needing to replace `my-release-cockroachdb-public` with the name of the `-public` Service that was created with your installed chart): 334 | 335 | ```shell 336 | kubectl run cockroach-client --rm -it \ 337 | --image=cockroachdb/cockroach \ 338 | --restart=Never \ 339 | -- sql --insecure --host my-release-cockroachdb-public 340 | ``` 341 | 342 | ``` 343 | Waiting for pod default/cockroach-client to be running, status is Pending, 344 | pod ready: false 345 | If you don't see a command prompt, try pressing enter. 346 | root@my-release-cockroachdb-public:26257> SHOW DATABASES; 347 | +--------------------+ 348 | | Database | 349 | +--------------------+ 350 | | information_schema | 351 | | pg_catalog | 352 | | system | 353 | +--------------------+ 354 | (3 rows) 355 | root@my-release-cockroachdb-public:26257> CREATE DATABASE bank; 356 | CREATE DATABASE 357 | root@my-release-cockroachdb-public:26257> CREATE TABLE bank.accounts (id INT 358 | PRIMARY KEY, balance DECIMAL); 359 | CREATE TABLE 360 | root@my-release-cockroachdb-public:26257> INSERT INTO bank.accounts VALUES 361 | (1234, 10000.50); 362 | INSERT 1 363 | root@my-release-cockroachdb-public:26257> SELECT * FROM bank.accounts; 364 | +------+---------+ 365 | | id | balance | 366 | +------+---------+ 367 | | 1234 | 10000.5 | 368 | +------+---------+ 369 | (1 row) 370 | root@my-release-cockroachdb-public:26257> \q 371 | Waiting for pod default/cockroach-client to terminate, status is Running 372 | pod "cockroach-client" deleted 373 | ``` 374 | 375 | > If you are running in secure mode, you will have to provide a client certificate to the cluster in order to authenticate, so the above command will not work. See [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) for an example of how to set up an interactive SQL shell against a secure cluster or [here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/example-app-secure.yaml) for an example application connecting to a secure cluster. 376 | 377 | ### Cluster health 378 | 379 | Because our pod spec includes regular health checks of the CockroachDB processes, simply running `kubectl get pods` and looking at the `STATUS` column is sufficient to determine the health of each instance in the cluster. 380 | 381 | If you want more detailed information about the cluster, the best place to look is the Admin UI. 382 | 383 | ### Accessing the Admin UI 384 | 385 | If you want to see information about how the cluster is doing, you can try pulling up the CockroachDB Admin UI by port-forwarding from your local machine to one of the pods (replacing `my-release-cockroachdb-0` with the name of one of your pods: 386 | 387 | ```shell 388 | kubectl port-forward my-release-cockroachdb-0 8080 389 | ``` 390 | 391 | You should then be able to access the Admin UI by visiting in your web browser. 392 | 393 | ### Failover 394 | 395 | If any CockroachDB member fails, it is restarted or recreated automatically by the Kubernetes infrastructure, and will re-join the cluster automatically when it comes back up. You can test this scenario by killing any of the CockroachDB pods: 396 | 397 | ```shell 398 | kubectl delete pod my-release-cockroachdb-1 399 | ``` 400 | 401 | ```shell 402 | kubectl get pods -l "app.kubernetes.io/instance=my-release,app.kubernetes.io/component=cockroachdb" 403 | ``` 404 | 405 | ``` 406 | NAME READY STATUS RESTARTS AGE 407 | my-release-cockroachdb-0 1/1 Running 0 5m 408 | my-release-cockroachdb-2 1/1 Running 0 5m 409 | ``` 410 | 411 | After a while: 412 | 413 | ```shell 414 | kubectl get pods -l "app.kubernetes.io/instance=my-release,app.kubernetes.io/component=cockroachdb" 415 | ``` 416 | 417 | ``` 418 | NAME READY STATUS RESTARTS AGE 419 | my-release-cockroachdb-0 1/1 Running 0 5m 420 | my-release-cockroachdb-1 1/1 Running 0 20s 421 | my-release-cockroachdb-2 1/1 Running 0 5m 422 | ``` 423 | 424 | You can check the state of re-joining from the new pod's logs: 425 | 426 | ```shell 427 | kubectl logs my-release-cockroachdb-1 428 | ``` 429 | 430 | ``` 431 | [...] 432 | I161028 19:32:09.754026 1 server/node.go:586 [n1] node connected via gossip and 433 | verified as part of cluster {"35ecbc27-3f67-4e7d-9b8f-27c31aae17d6"} 434 | [...] 435 | cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local:26257 436 | build: beta-20161027-55-gd2d3c7f @ 2016/10/28 19:27:25 (go1.7.3) 437 | admin: http://0.0.0.0:8080 438 | sql: 439 | postgresql://root@my-release-cockroachdb-1.my-release-cockroachdb.default.svc.cluster.local:26257?sslmode=disable 440 | logs: cockroach-data/logs 441 | store[0]: path=cockroach-data 442 | status: restarted pre-existing node 443 | clusterID: {35ecbc27-3f67-4e7d-9b8f-27c31aae17d6} 444 | nodeID: 2 445 | [...] 446 | ``` 447 | 448 | ### NetworkPolicy 449 | 450 | To enable NetworkPolicy for CockroachDB, install [a networking plugin that implements the Kubernetes NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), and set `networkPolicy.enabled` to `yes`/`true`. 451 | 452 | For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting the `DefaultDeny` Namespace annotation. Note: this will enforce policy for _all_ pods in the Namespace: 453 | 454 | ```shell 455 | kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" 456 | ``` 457 | 458 | For more precise policy, set `networkPolicy.ingress.grpc` and `networkPolicy.ingress.http` rules. This will only allow pods that match the provided rules to connect to CockroachDB. 459 | 460 | ### Scaling 461 | 462 | Scaling should be managed via the `helm upgrade` command. After resizing your cluster on your cloud environment (e.g., GKE or EKS), run the following command to add a pod. This assumes you scaled from 3 to 4 nodes: 463 | 464 | ```shell 465 | kubectl patch cockroachdbs example -p '{"spec": {"statefulset":{"replicas":"4"}}}' --type merge 466 | ``` 467 | 468 | Note, that if you are running in secure mode (`tls.enabled` is `yes`/`true`) and increase the size of your cluster, you will also have to approve the CSR (certificate-signing request) of each new node (using `kubectl get csr` and `kubectl certificate approve`). 469 | 470 | [1]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity 471 | [2]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity 472 | [3]: https://cert-manager.io/ 473 | [4]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass 474 | [5]: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ 475 | displayName: CockroachDB 476 | icon: 477 | - base64data: PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAzMS44MiAzMiIgd2lkdGg9IjI0ODYiIGhlaWdodD0iMjUwMCI+PHRpdGxlPkNMPC90aXRsZT48cGF0aCBkPSJNMTkuNDIgOS4xN2ExNS4zOSAxNS4zOSAwIDAgMS0zLjUxLjQgMTUuNDYgMTUuNDYgMCAwIDEtMy41MS0uNCAxNS42MyAxNS42MyAwIDAgMSAzLjUxLTMuOTEgMTUuNzEgMTUuNzEgMCAwIDEgMy41MSAzLjkxek0zMCAuNTdBMTcuMjIgMTcuMjIgMCAwIDAgMjUuNTkgMGExNy40IDE3LjQgMCAwIDAtOS42OCAyLjkzQTE3LjM4IDE3LjM4IDAgMCAwIDYuMjMgMGExNy4yMiAxNy4yMiAwIDAgMC00LjQ0LjU3QTE2LjIyIDE2LjIyIDAgMCAwIDAgMS4xM2EuMDcuMDcgMCAwIDAgMCAuMDkgMTcuMzIgMTcuMzIgMCAwIDAgLjgzIDEuNTcuMDcuMDcgMCAwIDAgLjA4IDAgMTYuMzkgMTYuMzkgMCAwIDEgMS44MS0uNTQgMTUuNjUgMTUuNjUgMCAwIDEgMTEuNTkgMS44OCAxNy41MiAxNy41MiAwIDAgMC0zLjc4IDQuNDhjLS4yLjMyLS4zNy42NS0uNTUgMXMtLjIyLjQ1LS4zMy42OS0uMzEuNzItLjQ0IDEuMDhhMTcuNDYgMTcuNDYgMCAwIDAgNC4yOSAxOC43Yy4yNi4yNS41My40OS44MS43M3MuNDQuMzcuNjcuNTQuNTkuNDQuODkuNjRhLjA3LjA3IDAgMCAwIC4wOCAwYy4zLS4yMS42LS40Mi44OS0uNjRzLjQ1LS4zNS42Ny0uNTQuNTUtLjQ4LjgxLS43M2ExNy40NSAxNy40NSAwIDAgMCA1LjM4LTEyLjYxIDE3LjM5IDE3LjM5IDAgMCAwLTEuMDktNi4wOWMtLjE0LS4zNy0uMjktLjczLS40NS0xLjA5cy0uMjItLjQ3LS4zMy0uNjktLjM1LS42Ni0uNTUtMWExNy42MSAxNy42MSAwIDAgMC0zLjc4LTQuNDggMTUuNjUgMTUuNjUgMCAwIDEgMTEuNi0xLjg0IDE2LjEzIDE2LjEzIDAgMCAxIDEuODEuNTQuMDcuMDcgMCAwIDAgLjA4IDBxLjQ0LS43Ni44Mi0xLjU2YS4wNy4wNyAwIDAgMCAwLS4wOUExNi44OSAxNi44OSAwIDAgMCAzMCAuNTd6IiBmaWxsPSIjMTUxZjM0Ii8+PHBhdGggZD0iTTIxLjgyIDE3LjQ3YTE1LjUxIDE1LjUxIDAgMCAxLTQuMjUgMTAuNjkgMTUuNjYgMTUuNjYgMCAwIDEtLjcyLTQuNjggMTUuNSAxNS41IDAgMCAxIDQuMjUtMTAuNjkgMTUuNjIgMTUuNjIgMCAwIDEgLjcyIDQuNjgiIGZpbGw9IiMzNDg1NDAiLz48cGF0aCBkPSJNMTUgMjMuNDhhMTUuNTUgMTUuNTUgMCAwIDEtLjcyIDQuNjggMTUuNTQgMTUuNTQgMCAwIDEtMy41My0xNS4zN0ExNS41IDE1LjUgMCAwIDEgMTUgMjMuNDgiIGZpbGw9IiM3ZGJjNDIiLz48L3N2Zz4= 478 | mediatype: image/svg+xml 479 | install: 480 | spec: 481 | deployments: null 482 | strategy: "" 483 | installModes: 484 | - supported: true 485 | type: OwnNamespace 486 | - supported: true 487 | type: SingleNamespace 488 | - supported: false 489 | type: MultiNamespace 490 | - supported: true 491 | type: AllNamespaces 492 | keywords: 493 | - cockroach 494 | - cockroachdb 495 | links: 496 | - name: Helm Chart Source 497 | url: https://github.com/helm/charts/tree/master/stable/cockroachdb 498 | - name: Configuration Options 499 | url: https://github.com/helm/charts/tree/master/stable/cockroachdb#configuration 500 | - name: CockroachDB Source 501 | url: https://github.com/cockroachdb/cockroach 502 | maintainers: 503 | - email: dmesser@redhat.com 504 | name: Daniel Messer 505 | maturity: stable 506 | minKubeVersion: 1.8.0 507 | provider: 508 | name: Helm Community 509 | url: https://artifacthub.io/packages/helm/cockroachdb/cockroachdb 510 | version: 5.0.4 511 | --------------------------------------------------------------------------------