├── .github └── workflows │ └── release-chart.yml ├── LICENSE ├── README.md ├── charts ├── deepflow-agent │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ │ ├── NOTES.txt │ │ ├── RBAC.yaml │ │ ├── _affinity.tpl │ │ ├── _helpers.tpl │ │ ├── configmap.yaml │ │ ├── daemonset.yaml │ │ ├── service.yaml │ │ └── watcher-deployment.yaml │ └── values.yaml └── deepflow │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── charts │ ├── byconity │ │ ├── Chart.yaml │ │ ├── charts │ │ │ ├── fdb-operator │ │ │ │ ├── Chart.yaml │ │ │ │ ├── crds │ │ │ │ │ ├── apps.foundationdb.org_foundationdbbackups.yaml │ │ │ │ │ ├── apps.foundationdb.org_foundationdbclusters.yaml │ │ │ │ │ └── apps.foundationdb.org_foundationdbrestores.yaml │ │ │ │ ├── templates │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ ├── manager │ │ │ │ │ │ └── deployment.yaml │ │ │ │ │ ├── rbac │ │ │ │ │ │ ├── rbac_role.yaml │ │ │ │ │ │ ├── rbac_role_binding.yaml │ │ │ │ │ │ └── serviceaccount.yaml │ │ │ │ │ └── tests │ │ │ │ │ │ └── test-connection.yaml │ │ │ │ └── values.yaml │ │ │ └── hdfs │ │ │ │ ├── .helmignore │ │ │ │ ├── Chart.yaml │ │ │ │ ├── README.md │ │ │ │ ├── docs │ │ │ │ ├── aws-eks-deployment.md │ │ │ │ └── kind-deployment.md │ │ │ │ ├── templates │ │ │ │ ├── _helpers.tpl │ │ │ │ ├── config.yaml │ │ │ │ ├── datanodes-netpol.yaml │ │ │ │ ├── datanodes-service.yaml │ │ │ │ ├── datanodes.yaml │ │ │ │ ├── ingress.yaml │ │ │ │ ├── namenode-netpol.yaml │ │ │ │ ├── namenode-service.yaml │ │ │ │ ├── namenode-serviceaccount.yaml │ │ │ │ ├── namenode.yaml │ │ │ │ ├── post-install-cmds.yaml │ │ │ │ ├── rack-awareness-rbac.yaml │ │ │ │ ├── rack-awareness-script.yaml │ │ │ │ ├── secrets.yaml │ │ │ │ ├── shell.yaml │ │ │ │ └── tests │ │ │ │ │ ├── auth-test.yaml │ │ │ │ │ ├── rack-awareness-test.yaml │ │ │ │ │ └── webhdfs-test.yaml │ │ │ │ ├── values-eks-alb.yaml │ │ │ │ ├── values-host-based-ingress.yaml │ │ │ │ └── values.yaml │ │ ├── files │ │ │ ├── cnch-config.yaml │ │ │ ├── daemon-manager.yaml │ │ │ ├── resource-manager.yaml │ │ │ ├── server.yaml │ │ │ ├── tso.yaml │ │ │ ├── users.yaml │ │ │ └── worker.yaml │ │ ├── templates │ │ │ ├── NOTES.txt │ │ │ ├── _helpers.tpl │ │ │ ├── daemonmanager-configmap.yaml │ │ │ ├── daemonmanager-service.yaml │ │ │ ├── daemonmanager.yaml │ │ │ ├── fdb-configmap.yaml │ │ │ ├── fdb_cluster.yaml │ │ │ ├── fdb_shell.yaml │ │ │ ├── ingress.yaml │ │ │ ├── lifecycle.yaml │ │ │ ├── resourcemanager-configmap.yaml │ │ │ ├── resourcemanager-service.yaml │ │ │ ├── resourcemanager.yaml │ │ │ ├── server-configmap.yaml │ │ │ ├── server-service.yaml │ │ │ ├── server.yaml │ │ │ ├── tso-configmap.yaml │ │ │ ├── tso-service.yaml │ │ │ ├── tso.yaml │ │ │ ├── vw-configmap.yaml │ │ │ ├── vw-service.yaml │ │ │ └── vw.yaml │ │ └── values.yaml │ ├── clickhouse │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ │ ├── _affinity.tpl │ │ │ ├── _helpers.tpl │ │ │ ├── _volumeClaimTemplates.tpl │ │ │ ├── configmap.yaml │ │ │ ├── service.yaml │ │ │ └── statefulset.yaml │ │ └── values.yaml │ ├── mysql │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ │ ├── _affinity.tpl │ │ │ ├── _helpers.tpl │ │ │ ├── configmap.yaml │ │ │ ├── deployment.yaml │ │ │ ├── pvc.yaml │ │ │ └── service.yaml │ │ └── values.yaml │ └── stella-agent-ce │ │ ├── Chart.yaml │ │ ├── templates │ │ ├── _affinity.tpl │ │ ├── _helpers.tpl │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ └── service.yaml │ │ └── values.yaml │ ├── templates │ ├── NOTES.txt │ ├── RBAC.yaml │ ├── _affinity_app.tpl │ ├── _affinity_server.tpl │ ├── _helpers.tpl │ ├── app-deployment.yaml │ ├── configmap.yaml │ ├── external-clickhouse-svc.yaml │ ├── server-deployment.yaml │ └── service.yaml │ └── values.yaml ├── commit-template ├── deepflow.png ├── deepflow.svg └── test-values.yaml /README.md: -------------------------------------------------------------------------------- 1 | # DeepFlow Helm Charts 2 | 3 | An automated observability platform for cloud-native developers. 4 | 5 | This repository contains [Helm](https://helm.sh/) charts for DeepFlow project. 6 | 7 | ## Usage 8 | 9 | [Helm](https://helm.sh) must be installed to use the charts. 10 | Please refer to Helm's [documentation](https://helm.sh/docs/) to get started. 11 | 12 | Once Helm is set up properly, add the repo as follows: 13 | 14 | ```console 15 | helm repo add deepflow https://deepflowio.github.io/deepflow 16 | helm repo update deepflow 17 | ``` 18 | 19 | ## Helm Charts 20 | 21 | You can then run `helm search repo deepflow` to see the charts. 22 | 23 | _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ 24 | 25 | ## Installing the Chart 26 | 27 | To install the chart with the release name `deepflow`: 28 | 29 | ```console 30 | helm install deepflow -n deepflow deepflow/deepflow --create-namespace 31 | ``` 32 | 33 | ## Uninstalling the Chart 34 | 35 | To uninstall/delete the my-release deployment: 36 | 37 | ```console 38 | helm delete deepflow -n deepflow 39 | ``` 40 | 41 | The command removes all the Kubernetes components associated with the chart and deletes the release. 42 | 43 | 44 | ## License 45 | 46 | [Apache 2.0 License](./LICENSE). -------------------------------------------------------------------------------- /charts/deepflow-agent/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/deepflow-agent/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | description: An automated observability platform for cloud-native developers. 3 | name: deepflow-agent 4 | engine: gotpl 5 | type: application 6 | version: 7.0.008 7 | appVersion: "7.0.8" 8 | kubeVersion: ">=1.16.0-0" 9 | sources: 10 | - https://github.com/deepflowio/deepflow 11 | home: https://github.com/deepflowio/deepflow 12 | keywords: 13 | - deepflow 14 | icon: https://raw.githubusercontent.com/deepflowio/deepflow-charts/main/deepflow.svg 15 | maintainers: 16 | - name: deepflow 17 | url: https://github.com/deepflowio/deepflow 18 | -------------------------------------------------------------------------------- /charts/deepflow-agent/README.md: -------------------------------------------------------------------------------- 1 | # DeepFlow Agent Helm Charts 2 | 3 | 4 | This repository contains [Helm](https://helm.sh/) charts for DeepFlow Agent project. 5 | 6 | ## Usage 7 | 8 | ### Prerequisites 9 | 10 | - Kubernetes 1.16+ 11 | - Helm 3+ 12 | 13 | [Helm](https://helm.sh) must be installed to use the charts. 14 | Please refer to Helm's [documentation](https://helm.sh/docs/) to get started. 15 | 16 | Once Helm is set up properly, add the repo as follows: 17 | 18 | ```console 19 | helm repo add deepflow https://deepflowio.github.io/deepflow 20 | helm repo udpate deepflow 21 | ``` 22 | 23 | ## Helm Charts 24 | 25 | You can then run `helm search repo deepflow-agent` to see the charts. 26 | 27 | _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ 28 | 29 | ## Installing the Chart 30 | 31 | To install the chart with the release name `deepflow-agent`: 32 | 33 | ```console 34 | helm install deepflow-agent -n deepflow deepflow/deepflow-agent --create-namespace 35 | ``` 36 | 37 | ## Uninstalling the Chart 38 | 39 | To uninstall/delete the my-release deployment: 40 | 41 | ```console 42 | helm delete deepflow-agent -n deepflow 43 | ``` 44 | 45 | The command removes all the Kubernetes components associated with the chart and deletes the release. 46 | 47 | 48 | 49 | ## Main values block usage: 50 | 51 | ### Common 52 | 53 | ```yaml 54 | deepflowServerNodeIPS: 55 | - deepflow-server # The IP address of the Deepflow server node 56 | deepflowK8sClusterID: e70999ed-fdff-4277-be3c-4a3fceae215f # The ID of the Deepflow Kubernetes cluster 57 | agentGroupID: 1 # The ID of the agent group 58 | controllerPort: 443 # The port of the kubernetes apiserver 59 | clusterNAME: worker # # The name of the cluster 60 | ``` 61 | 62 | 63 | ### Affinity: 64 | 65 | The affinity of component. Combine `global.affinity` by 'OR'. 66 | 67 | - podAntiAffinityLabelSelector: affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution 68 | 69 | ```yaml 70 | podAntiAffinityLabelSelector: 71 | - labelSelector: 72 | - key: app #your label key 73 | operator: In # In、NotIn、Exists、 DoesNotExist 74 | values: deepflow #your label value, Multiple values separated by commas 75 | - key: component 76 | operator: In 77 | values: deepflow-server,deepflowys 78 | topologyKey: "kubernetes.io/hostname" 79 | ``` 80 | 81 | - podAntiAffinityTermLabelSelector: affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution 82 | 83 | ```yaml 84 | podAntiAffinityLabelSelector: 85 | - labelSelector: 86 | - key: app # your label key 87 | operator: In # In、NotIn、Exists、 DoesNotExist 88 | values: deepflow # your label value, Multiple values separated by commas 89 | - key: component 90 | operator: In 91 | values: deepflow-server,deepflowys 92 | topologyKey: "kubernetes.io/hostname" 93 | ``` 94 | 95 | - podAffinityLabelSelector: affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution 96 | 97 | ```yaml 98 | podAffinityLabelSelector: 99 | - labelSelector: 100 | - key: app 101 | operator: In 102 | values: deepflow 103 | - key: component 104 | operator: In 105 | values: clickhouse 106 | topologyKey: "kubernetes.io/hostname" 107 | ``` 108 | 109 | - podAffinityTermLabelSelector: affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution 110 | 111 | ```yaml 112 | podAffinityTermLabelSelector: 113 | - topologyKey: kubernetes.io/hostname 114 | weight: 10 115 | labelSelector: 116 | - key: app 117 | operator: In 118 | values: deepflow,deepflowys 119 | ``` 120 | 121 | - nodeAffinityLabelSelector: affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution 122 | 123 | ```yaml 124 | nodeAffinityLabelSelector: 125 | - matchExpressions: 126 | - key: app 127 | operator: In 128 | values: deepflow,deepflowys 129 | ``` 130 | 131 | - nodeAffinityTermLabelSelector: affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution 132 | 133 | ```yaml 134 | nodeAffinityTermLabelSelector: 135 | - weight: 10 136 | matchExpressions: 137 | - key: app 138 | operator: In 139 | values: deepflow,deepflowys 140 | ``` -------------------------------------------------------------------------------- /charts/deepflow-agent/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | ██████╗ ███████╗███████╗██████╗ ███████╗██╗ ██████╗ ██╗ ██╗ 2 | ██╔══██╗██╔════╝██╔════╝██╔══██╗██╔════╝██║ ██╔═══██╗██║ ██║ 3 | ██║ ██║█████╗ █████╗ ██████╔╝█████╗ ██║ ██║ ██║██║ █╗ ██║ 4 | ██║ ██║██╔══╝ ██╔══╝ ██╔═══╝ ██╔══╝ ██║ ██║ ██║██║███╗██║ 5 | ██████╔╝███████╗███████╗██║ ██║ ███████╗╚██████╔╝╚███╔███╔╝ 6 | ╚═════╝ ╚══════╝╚══════╝╚═╝ ╚═╝ ╚══════╝ ╚═════╝ ╚══╝╚══╝ 7 | 8 | An automated observability platform for cloud-native developers. 9 | 10 | # deepflow-agent Port for receiving trace, metrics, and log 11 | 12 | deepflow-agent service: {{ include "deepflow-agent.fullname" . }}.{{ $.Release.Namespace }} 13 | deepflow-agent Host listening port: {{ tpl (toString .Values.externalAgentHttpProxyPort) . }} 14 | -------------------------------------------------------------------------------- /charts/deepflow-agent/templates/RBAC.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "deepflow-agent.fullname" . }} 5 | --- 6 | kind: ClusterRole 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | metadata: 9 | name: {{ include "deepflow-agent.fullname" . }} 10 | namespace: {{ .Release.Namespace }} 11 | rules: 12 | - apiGroups: [""] 13 | resources: 14 | - nodes 15 | - namespaces 16 | - configmaps 17 | - services 18 | - pods 19 | - replicationcontrollers 20 | verbs: ["get", "list", "watch"] 21 | - apiGroups: ["apps"] 22 | resources: 23 | - daemonsets 24 | - deployments 25 | - replicasets 26 | - statefulsets 27 | verbs: ["get", "list", "watch"] 28 | - apiGroups: ["extensions", "networking.k8s.io"] 29 | resources: ["ingresses"] 30 | verbs: ["get", "list", "watch"] 31 | - apiGroups: ["route.openshift.io"] 32 | resources: ["routes"] 33 | verbs: ["get", "list", "watch"] 34 | --- 35 | kind: ClusterRoleBinding 36 | apiVersion: rbac.authorization.k8s.io/v1 37 | metadata: 38 | name: {{ include "deepflow-agent.fullname" . }} 39 | subjects: 40 | - kind: ServiceAccount 41 | name: {{ include "deepflow-agent.fullname" . }} 42 | namespace: {{ .Release.Namespace }} 43 | apiGroup: "" 44 | roleRef: 45 | kind: ClusterRole 46 | name: {{ include "deepflow-agent.fullname" . }} 47 | apiGroup: "" 48 | --- 49 | ### DeepFlow Agent remote update is used only to update DeepFlow-Agent's daemonset and configmap 50 | kind: Role 51 | apiVersion: rbac.authorization.k8s.io/v1 52 | metadata: 53 | name: {{ include "deepflow-agent.fullname" . }}-update 54 | namespace: {{ .Release.Namespace }} 55 | rules: 56 | - apiGroups: [""] 57 | resources: 58 | - configmaps 59 | resourceNames: ['{{ include "deepflow-agent.fullname" . }}-config'] 60 | verbs: ["get", "list", "watch", "update", "patch"] 61 | - apiGroups: ["apps"] 62 | resources: 63 | - daemonsets 64 | resourceNames: ['{{ include "deepflow-agent.fullname" . }}'] 65 | verbs: ["get", "list", "watch", "update", "patch"] 66 | --- 67 | apiVersion: rbac.authorization.k8s.io/v1 68 | kind: RoleBinding 69 | metadata: 70 | name: {{ include "deepflow-agent.fullname" . }}-update 71 | namespace: {{ .Release.Namespace }} 72 | roleRef: 73 | apiGroup: rbac.authorization.k8s.io 74 | kind: Role 75 | name: {{ include "deepflow-agent.fullname" . }}-update 76 | subjects: 77 | - kind: ServiceAccount 78 | name: {{ include "deepflow-agent.fullname" . }} 79 | apiGroup: "" -------------------------------------------------------------------------------- /charts/deepflow-agent/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "deepflow-agent.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "deepflow-agent.fullname" -}} 14 | {{- if .Values.agentFullnameOverride }} 15 | {{- .Values.agentFullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "deepflow-agent.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "deepflow-agent.labels" -}} 37 | helm.sh/chart: {{ include "deepflow-agent.chart" . }} 38 | {{ include "deepflow-agent.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "deepflow-agent.selectorLabels" -}} 49 | app: deepflow 50 | component: deepflow-agent 51 | app.kubernetes.io/name: {{ include "deepflow-agent.name" . }} 52 | app.kubernetes.io/instance: {{ .Release.Name }} 53 | {{- end }} 54 | 55 | {{/* 56 | Create the name of the service account to use 57 | */}} 58 | {{- define "deepflow-agent.serviceAccountName" -}} 59 | {{- if .Values.serviceAccount.create }} 60 | {{- default (include "deepflow-agent.fullname" .) .Values.serviceAccount.name }} 61 | {{- else }} 62 | {{- default "default" .Values.serviceAccount.name }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/deepflow-agent/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: {{ include "deepflow-agent.fullname" . }} 5 | labels: 6 | {{- include "deepflow-agent.labels" . | nindent 4 }} 7 | data: 8 | {{- tpl (toYaml .Values.deepflowAgentConfig) $ | nindent 2 }} -------------------------------------------------------------------------------- /charts/deepflow-agent/templates/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: {{ include "deepflow-agent.fullname" . }} 7 | labels: 8 | {{- include "deepflow-agent.labels" . | nindent 4 }} 9 | {{- if .Values.service.annotations }} 10 | annotations: 11 | {{ toYaml .Values.service.annotations | indent 4 }} 12 | {{- end }} 13 | spec: 14 | {{- if .Values.service.clusterIP }} 15 | clusterIP: {{ .Values.service.clusterIP }} 16 | {{- end }} 17 | {{- if .Values.service.externalIPs }} 18 | externalIPs: 19 | {{ toYaml .Values.service.externalIPs | indent 4 }} 20 | {{- end }} 21 | {{- if .Values.service.loadBalancerIP }} 22 | loadBalancerIP: {{ .Values.service.loadBalancerIP }} 23 | {{- end }} 24 | {{- if .Values.service.loadBalancerSourceRanges }} 25 | loadBalancerSourceRanges: 26 | {{- range $cidr := .Values.service.loadBalancerSourceRanges }} 27 | - {{ $cidr }} 28 | {{- end }} 29 | {{- end }} 30 | {{- if ne .Values.service.type "ClusterIP" }} 31 | externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} 32 | {{- end }} 33 | ports: 34 | {{- $serviceType := .Values.service.type -}} 35 | {{- range .Values.service.ports }} 36 | - name: {{ tpl .name $ }} 37 | port: {{ tpl (toString .port) $ }} 38 | targetPort: {{ tpl (toString .targetPort) $ }} 39 | {{- if and (eq $serviceType "NodePort") (.nodePort) }} 40 | nodePort: {{ tpl (toString .nodePort) $ }} 41 | {{- end }} 42 | protocol: {{ tpl .protocol $ }} 43 | {{- end }} 44 | {{- if .Values.service.additionalPorts }} 45 | {{ toYaml .Values.service.additionalPorts | indent 2 }} 46 | {{- end }} 47 | selector: 48 | {{- include "deepflow-agent.selectorLabels" . | nindent 4 }} 49 | type: "{{ .Values.service.type }}" -------------------------------------------------------------------------------- /charts/deepflow-agent/templates/watcher-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if has "watcher" $.Values.deployComponent }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "deepflow-agent.fullname" . }}-watcher 6 | labels: 7 | {{- include "deepflow-agent.labels" . | nindent 4 }} 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | {{- include "deepflow-agent.selectorLabels" . | nindent 6 }} 13 | template: 14 | metadata: 15 | annotations: 16 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 17 | checksum/customConfig: {{ sha256sum (print .Values.deepflowAgentConfig) }} 18 | container.apparmor.security.beta.kubernetes.io/deepflow-agent: unconfined 19 | {{- with .Values.podAnnotations }} 20 | {{- toYaml . | nindent 8 }} 21 | {{- end }} 22 | labels: 23 | {{- include "deepflow-agent.selectorLabels" . | nindent 8 }} 24 | {{- with .Values.podLabels }} 25 | {{- toYaml . | nindent 8 }} 26 | {{- end }} 27 | spec: 28 | dnsPolicy: ClusterFirst 29 | imagePullSecrets: 30 | {{- with .Values.global.imagePullSecrets }} 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | {{- with .Values.imagePullSecrets }} 34 | {{- toYaml . | nindent 8 }} 35 | {{- end }} 36 | serviceAccountName: {{ include "deepflow-agent.fullname" . }} 37 | containers: 38 | - name: {{ .Chart.Name }} 39 | image: "{{ tpl .Values.image.repository . }}:{{ tpl (toString .Values.image.tag) . }}" 40 | imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} 41 | ports: 42 | - name: receive 43 | containerPort: {{ tpl (toString .Values.externalAgentHttpProxyPort) . }} 44 | protocol: TCP 45 | # livenessProbe: 46 | # tcpSocket: 47 | # port: otel 48 | # initialDelaySeconds: 5 49 | # periodSeconds: 10 50 | # readinessProbe: 51 | # tcpSocket: 52 | # port: otel 53 | # initialDelaySeconds: 5 54 | # periodSeconds: 10 55 | env: 56 | - name: K8S_WATCH_POLICY 57 | value: "watch-only" 58 | {{- range .Values.env}} 59 | - name: {{ .name }} 60 | value: {{ .value }} 61 | {{- end}} 62 | resources: 63 | {{- toYaml .Values.resources | nindent 12 }} 64 | volumeMounts: 65 | - name: deepflow-agent-conf 66 | mountPath: /etc/deepflow-agent 67 | volumes: 68 | - name: deepflow-agent-conf 69 | configMap: 70 | name: {{ include "deepflow-agent.fullname" . }} 71 | {{- with .Values.nodeSelector }} 72 | nodeSelector: 73 | {{- toYaml . | nindent 8 }} 74 | {{- end }} 75 | affinity: 76 | {{- include "nodeaffinity" . | indent 6 }} 77 | {{- include "podAffinity" . | indent 6 }} 78 | {{- include "podAntiAffinity" . | indent 6 }} 79 | {{- if or .Values.global.tolerations .Values.tolerations }} 80 | tolerations: 81 | {{- if .Values.global.tolerations }} 82 | {{- toYaml .Values.global.tolerations | nindent 8 }} 83 | {{- end }} 84 | {{- if .Values.tolerations }} 85 | {{- toYaml .Values.tolerations | nindent 8 }} 86 | {{- end }} 87 | {{- end }} 88 | {{- end }} 89 | -------------------------------------------------------------------------------- /charts/deepflow-agent/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for deepflow-agent. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | global: 5 | podAntiAffinityLabelSelector: [] 6 | podAntiAffinityTermLabelSelector: [] 7 | podAffinityLabelSelector: [] 8 | podAffinityTermLabelSelector: [] 9 | nodeAffinityLabelSelector: [] 10 | nodeAffinityTermLabelSelector: [] 11 | 12 | deployComponent: 13 | - "daemonset" 14 | # - "watcher" 15 | # 16 | tkeSidecar: false 17 | daemonsetWatchDisabled: false # Whether to disable the watch for Agent configured through daemonset 18 | 19 | image: 20 | repository: registry.cn-hongkong.aliyuncs.com/deepflow-ce/deepflow-agent 21 | pullPolicy: Always 22 | # Overrides the image tag whose default is the chart appVersion. 23 | tag: v7.0.8 24 | 25 | env: [] 26 | 27 | imagePullSecrets: [] 28 | nameOverride: "" 29 | agentFullnameOverride: "deepflow-agent" 30 | 31 | podAnnotations: {} 32 | ## Pod Labels 33 | podLabels: {} 34 | podSecurityContext: {} 35 | # fsGroup: 2000 36 | hostNetwork: "false" 37 | dnsPolicy: "ClusterFirst" 38 | 39 | ## Mount the netns hostPath directory read-only 40 | netns: 41 | mount: false 42 | 43 | securityContext: 44 | # privileged: true 45 | capabilities: 46 | add: 47 | - SYS_ADMIN ## Permission required for collecting K8s information and eBPF data 48 | - SYS_RESOURCE ## Permissions required to collect eBPF data 49 | - SYS_PTRACE ## Permission required for collecting K8s information 50 | - NET_ADMIN ## Indicates the permission for collecting AF_PACKET traffic 51 | - NET_RAW ## Indicates the permission for collecting AF_PACKET traffic 52 | - IPC_LOCK ## optional MAP_LOCKED MAP_NORESERVE: Indicates the permission for collecting AF_PACKET traffic, It can significantly degrade performance when not available 53 | - SYSLOG 54 | #- BPF ## Optionally, eBPF does not require SYS_ADMIN in kernel Linux 5.8+ and uses a combination of BPF and PERFMON instead 55 | #- PERFMON ## Optionally, eBPF does not require SYS_ADMIN in kernel Linux 5.8+ and uses a combination of BPF and PERFMON instead 56 | # readOnlyRootFilesystem: true 57 | # runAsNonRoot: true 58 | # runAsUser: 1000 59 | 60 | 61 | timezone: "Asia/Shanghai" 62 | 63 | ## File read and write permissions required for collecting eBPF data (Optional. If you do not have the permission and the content does not meet expectations, the performance will be significantly reduced. You can set the content in advance.) 64 | ## sysctl -w net.core.bpf_jit_enable=1 65 | sysctlInitContainer: 66 | enabled: true 67 | 68 | extraVolumeMounts: [] 69 | service: 70 | ## Configuration for Clickhouse service 71 | ## 72 | annotations: {} 73 | labels: {} 74 | clusterIP: "" 75 | 76 | ## Port for Clickhouse Service to listen on 77 | ## 78 | 79 | ports: 80 | - name: receive 81 | port: 80 82 | targetPort: receive 83 | nodePort: 84 | protocol: TCP 85 | 86 | ## Additional ports to open for server service 87 | additionalPorts: [] 88 | 89 | externalIPs: [] 90 | loadBalancerIP: "" 91 | loadBalancerSourceRanges: [] 92 | 93 | ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints 94 | externalTrafficPolicy: Cluster 95 | 96 | ## Service type 97 | ## 98 | type: ClusterIP 99 | 100 | deepflowServerNodeIPS: 101 | - deepflow-server 102 | deepflowK8sClusterID: 103 | agentGroupID: 104 | controllerPort: 105 | clusterNAME: 106 | teamId: 107 | ## This command takes effect when agent-group-config external_agent_http_proxy_enabled=1 is configured using deepflow-ctl 108 | ## Set the port to the agent-group-config port 109 | externalAgentHttpProxyPort: 38086 110 | 111 | deepflowAgentConfig: 112 | deepflow-agent.yaml: | 113 | {{- nindent 4 "controller-ips:" -}} 114 | {{- tpl (toYaml $.Values.deepflowServerNodeIPS ) .| nindent 4 -}} 115 | {{- if .Values.agentGroupID }}{{ nindent 4 "vtap-group-id-request:" }} {{ .Values.agentGroupID }}{{ end -}} 116 | {{- if .Values.deepflowK8sClusterID }}{{ nindent 4 "kubernetes-cluster-id:" }} {{ .Values.deepflowK8sClusterID }}{{ end -}} 117 | {{- if .Values.controllerPort }}{{ nindent 4 "controller-port:" }} {{ .Values.controllerPort }}{{ end -}} 118 | {{- if .Values.clusterNAME }}{{ nindent 4 "kubernetes-cluster-name:" }} {{ .Values.clusterNAME }}{{ end -}} 119 | {{- if .Values.teamId }}{{ nindent 4 "team-id:" }} {{ .Values.teamId }}{{ end -}} 120 | 121 | resources: 122 | limits: 123 | cpu: 1000m 124 | memory: 768Mi 125 | requests: 126 | cpu: 100m 127 | memory: 128Mi 128 | 129 | 130 | nodeSelector: {} 131 | 132 | tolerations: [] 133 | # - key: "key1" 134 | # operator: "Equal" 135 | # value: "value1" 136 | # effect: "NoSchedule" 137 | 138 | podAntiAffinityLabelSelector: [] 139 | podAntiAffinityTermLabelSelector: [] 140 | podAffinityLabelSelector: [] 141 | podAffinityTermLabelSelector: [] 142 | nodeAffinityLabelSelector: [] 143 | nodeAffinityTermLabelSelector: [] 144 | 145 | -------------------------------------------------------------------------------- /charts/deepflow/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/deepflow/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | description: An automated observability platform for cloud-native developers. 3 | name: deepflow 4 | engine: gotpl 5 | type: application 6 | version: 7.0.008 7 | appVersion: "7.0.8" 8 | kubeVersion: ">=1.16.0-0" 9 | sources: 10 | - https://github.com/deepflowio/deepflow 11 | home: https://github.com/deepflowio/deepflow 12 | keywords: 13 | - deepflow 14 | icon: https://raw.githubusercontent.com/deepflowio/deepflow-charts/main/deepflow.svg 15 | maintainers: 16 | - name: deepflow 17 | url: https://github.com/deepflowio/deepflow 18 | dependencies: 19 | - name: clickhouse 20 | repository: "" 21 | version: '*.*.*' 22 | condition: clickhouse.enabled 23 | - name: mysql 24 | repository: "" 25 | version: '*.*.*' 26 | condition: mysql.enabled 27 | - name: deepflow-agent 28 | repository: "" 29 | version: '*.*.*' 30 | condition: deepflow-agent.enabled 31 | - name: grafana 32 | version: "7.*.*" 33 | repository: https://grafana.github.io/helm-charts 34 | condition: grafana.enabled 35 | - name: stella-agent-ce 36 | repository: "" 37 | version: '*.*.*' 38 | condition: stella-agent-ce.enabled 39 | - name: byconity 40 | repository: "" 41 | version: '*.*.*' 42 | condition: byconity.enabled -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: byconity 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | 26 | dependencies: 27 | - name: hdfs 28 | condition: hdfs.enabled 29 | version: '*.*.*' 30 | - name: fdb-operator 31 | condition: fdb-operator.enabled 32 | version: '*.*.*' 33 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: fdb-operator 3 | description: A Helm chart for foundationDB operator 4 | home: https://www.foundationdb.org/ 5 | sources: 6 | - https://github.com/FoundationDB/fdb-kubernetes-operator/tree/master/helm/fdb-operator 7 | 8 | # A chart can be either an 'application' or a 'library' chart. 9 | # 10 | # Application charts are a collection of templates that can be packaged into versioned archives 11 | # to be deployed. 12 | # 13 | # Library charts provide useful utilities or functions for the chart developer. They're included as 14 | # a dependency of application charts to inject those utilities and functions into the rendering 15 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 16 | type: application 17 | 18 | # This is the chart version. This version number should be incremented each time you make changes 19 | # to the chart and its templates, including the app version. 20 | version: 0.2.0 21 | 22 | # This is the version number of the application being deployed. This version number should be 23 | # incremented each time you make changes to the application. 24 | appVersion: v1.9.0 25 | 26 | maintainers: 27 | - name: "foundationdb-ci" 28 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/crds/apps.foundationdb.org_foundationdbrestores.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.9.2 7 | creationTimestamp: null 8 | name: foundationdbrestores.apps.foundationdb.org 9 | spec: 10 | group: apps.foundationdb.org 11 | names: 12 | kind: FoundationDBRestore 13 | listKind: FoundationDBRestoreList 14 | plural: foundationdbrestores 15 | shortNames: 16 | - fdbrestore 17 | singular: foundationdbrestore 18 | scope: Namespaced 19 | versions: 20 | - additionalPrinterColumns: 21 | - jsonPath: .metadata.creationTimestamp 22 | name: Age 23 | type: date 24 | deprecated: true 25 | name: v1beta1 26 | schema: 27 | openAPIV3Schema: 28 | properties: 29 | apiVersion: 30 | type: string 31 | kind: 32 | type: string 33 | metadata: 34 | type: object 35 | spec: 36 | properties: 37 | backupURL: 38 | type: string 39 | blobStoreConfiguration: 40 | properties: 41 | accountName: 42 | maxLength: 100 43 | type: string 44 | backupName: 45 | maxLength: 1024 46 | type: string 47 | bucket: 48 | maxLength: 63 49 | minLength: 3 50 | type: string 51 | urlParameters: 52 | items: 53 | maxLength: 1024 54 | type: string 55 | maxItems: 100 56 | type: array 57 | required: 58 | - accountName 59 | type: object 60 | customParameters: 61 | items: 62 | maxLength: 100 63 | type: string 64 | maxItems: 100 65 | type: array 66 | destinationClusterName: 67 | type: string 68 | keyRanges: 69 | items: 70 | properties: 71 | end: 72 | pattern: ^[A-Za-z0-9\/\\-]+$ 73 | type: string 74 | start: 75 | pattern: ^[A-Za-z0-9\/\\-]+$ 76 | type: string 77 | required: 78 | - end 79 | - start 80 | type: object 81 | type: array 82 | required: 83 | - destinationClusterName 84 | type: object 85 | status: 86 | properties: 87 | running: 88 | type: boolean 89 | type: object 90 | type: object 91 | served: true 92 | storage: false 93 | subresources: 94 | status: {} 95 | - additionalPrinterColumns: 96 | - jsonPath: .metadata.creationTimestamp 97 | name: Age 98 | type: date 99 | name: v1beta2 100 | schema: 101 | openAPIV3Schema: 102 | properties: 103 | apiVersion: 104 | type: string 105 | kind: 106 | type: string 107 | metadata: 108 | type: object 109 | spec: 110 | properties: 111 | blobStoreConfiguration: 112 | properties: 113 | accountName: 114 | maxLength: 100 115 | type: string 116 | backupName: 117 | maxLength: 1024 118 | type: string 119 | bucket: 120 | maxLength: 63 121 | minLength: 3 122 | type: string 123 | urlParameters: 124 | items: 125 | maxLength: 1024 126 | type: string 127 | maxItems: 100 128 | type: array 129 | required: 130 | - accountName 131 | type: object 132 | customParameters: 133 | items: 134 | maxLength: 100 135 | type: string 136 | maxItems: 100 137 | type: array 138 | destinationClusterName: 139 | type: string 140 | keyRanges: 141 | items: 142 | properties: 143 | end: 144 | pattern: ^[A-Za-z0-9\/\\-]+$ 145 | type: string 146 | start: 147 | pattern: ^[A-Za-z0-9\/\\-]+$ 148 | type: string 149 | required: 150 | - end 151 | - start 152 | type: object 153 | type: array 154 | required: 155 | - destinationClusterName 156 | type: object 157 | status: 158 | properties: 159 | running: 160 | type: boolean 161 | type: object 162 | type: object 163 | served: true 164 | storage: true 165 | subresources: 166 | status: {} 167 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | FoundationDB operator has been installed successfully. 2 | 3 | To see the logs of the operator you can use below command 4 | kubectl logs deployment/{{ include "fdb-operator.fullname" . }} -n {{ .Release.Namespace }} -f 5 | 6 | Thanks for trying out FoundationDB helm chart. 7 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "fdb-operator.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 6 | {{- end -}} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "fdb-operator.fullname" -}} 14 | {{- if .Values.fullnameOverride -}} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 16 | {{- else -}} 17 | {{- $name := default .Chart.Name .Values.nameOverride -}} 18 | {{- if contains $name .Release.Name -}} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 20 | {{- else -}} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 22 | {{- end -}} 23 | {{- end -}} 24 | {{- end -}} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "fdb-operator.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 31 | {{- end -}} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "fdb-operator.labels" -}} 37 | helm.sh/chart: {{ include "fdb-operator.chart" . }} 38 | {{ include "fdb-operator.selectorLabels" . }} 39 | app.kubernetes.io/version: {{ .Values.image.tag | trimPrefix "v" | quote }} 40 | app.kubernetes.io/managed-by: {{ .Release.Service }} 41 | {{- end }} 42 | 43 | {{/* 44 | Selector labels 45 | */}} 46 | {{- define "fdb-operator.selectorLabels" -}} 47 | app.kubernetes.io/name: {{ include "fdb-operator.name" . }} 48 | app.kubernetes.io/instance: {{ .Release.Name }} 49 | {{- end }} 50 | 51 | {{/* 52 | Create the name of the service account 53 | */}} 54 | {{- define "fdb-operator.serviceAccountName" -}} 55 | {{- if .Values.serviceAccount.create -}} 56 | {{ default (include "fdb-operator.fullname" .) .Values.serviceAccount.name }} 57 | {{- else -}} 58 | {{ default "default" .Values.serviceAccount.name }} 59 | {{- end -}} 60 | {{- end -}} 61 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/manager/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "fdb-operator.fullname" . }} 6 | labels: 7 | {{- include "fdb-operator.labels" . | nindent 4 }} 8 | {{- with .Values.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | spec: 13 | {{- with .Values.replicas }} 14 | replicas: {{ . }} 15 | {{- end }} 16 | selector: 17 | matchLabels: 18 | {{- include "fdb-operator.selectorLabels" . | nindent 6 }} 19 | template: 20 | metadata: 21 | labels: 22 | {{- include "fdb-operator.selectorLabels" . | nindent 8 }} 23 | {{- with .Values.podLabels }} 24 | {{- toYaml . | nindent 8 }} 25 | {{- end }} 26 | {{- with .Values.podAnnotations }} 27 | annotations: 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | spec: 31 | serviceAccountName: {{ include "fdb-operator.serviceAccountName" . }} 32 | {{- with .Values.priorityClassName }} 33 | priorityClassName: {{ . }} 34 | {{- end }} 35 | securityContext: 36 | {{- toYaml .Values.securityContext | nindent 8 }} 37 | terminationGracePeriodSeconds: 10 38 | imagePullSecrets: 39 | {{- with .Values.global.imagePullSecrets }} 40 | {{- toYaml . | nindent 8 }} 41 | {{- end }} 42 | {{- with .Values.imagePullSecrets }} 43 | {{- toYaml . | nindent 8 }} 44 | {{- end }} 45 | initContainers: 46 | {{- $imageRepository := tpl $.Values.initContainerImage.repository . }} 47 | {{- range $version, $params := $.Values.initContainers }} 48 | - name: foundationdb-kubernetes-init-{{ $version | replace "." "-" }} 49 | image: "{{ $imageRepository }}:{{ $params.image.tag }}" 50 | imagePullPolicy: {{ $params.image.pullPolicy }} 51 | args: 52 | - "--copy-library" 53 | - "{{ $version }}" 54 | - "--copy-binary" 55 | - "fdbcli" 56 | - "--copy-binary" 57 | - "fdbbackup" 58 | - "--copy-binary" 59 | - "fdbrestore" 60 | - "--output-dir" 61 | - "/var/output-files/{{ regexFind "^[0-9.]+" $params.image.tag }}" 62 | - "--init-mode" 63 | volumeMounts: 64 | - name: fdb-binaries 65 | mountPath: /var/output-files 66 | resources: 67 | {{- toYaml $.Values.initContainersResources | nindent 10 }} 68 | securityContext: 69 | {{- toYaml $.Values.initContainerSecurityContext | nindent 10 }} 70 | {{- end }} 71 | containers: 72 | - name: manager 73 | image: "{{ tpl $.Values.image.repository . }}:{{ tpl $.Values.image.tag . }}" 74 | imagePullPolicy: {{ .Values.image.pullPolicy }} 75 | command: 76 | - /manager 77 | {{- if not .Values.globalMode.enabled }} 78 | env: 79 | - name: WATCH_NAMESPACE 80 | valueFrom: 81 | fieldRef: 82 | fieldPath: metadata.namespace 83 | {{- end }} 84 | ports: 85 | - containerPort: 8080 86 | name: metrics 87 | volumeMounts: 88 | - name: tmp 89 | mountPath: /tmp 90 | - name: logs 91 | mountPath: /var/log/fdb 92 | - name: fdb-binaries 93 | mountPath: /usr/bin/fdb 94 | securityContext: 95 | {{- toYaml .Values.containerSecurityContext | nindent 10 }} 96 | livenessProbe: 97 | httpGet: 98 | path: /metrics 99 | port: metrics 100 | resources: 101 | {{- toYaml .Values.resources | nindent 10 }} 102 | {{- with .Values.affinity }} 103 | affinity: 104 | {{- toYaml . | nindent 8 }} 105 | {{- end }} 106 | {{- with .Values.nodeSelector }} 107 | nodeSelector: 108 | {{- toYaml . | nindent 8 }} 109 | {{- end }} 110 | {{- with .Values.tolerations }} 111 | tolerations: 112 | {{- toYaml . | nindent 8 }} 113 | {{- end }} 114 | volumes: 115 | - name: tmp 116 | emptyDir: {} 117 | - name: logs 118 | emptyDir: {} 119 | - name: fdb-binaries 120 | emptyDir: {} 121 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/rbac/rbac_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | {{- if .Values.globalMode.enabled }} 4 | kind: ClusterRole 5 | {{- else }} 6 | kind: Role 7 | {{- end }} 8 | metadata: 9 | name: {{ include "fdb-operator.fullname" . }} 10 | labels: 11 | {{- include "fdb-operator.labels" . | nindent 4 }} 12 | rules: 13 | - apiGroups: 14 | - "" 15 | resources: 16 | - pods 17 | - configmaps 18 | - persistentvolumeclaims 19 | - events 20 | verbs: 21 | - get 22 | - watch 23 | - list 24 | - create 25 | - update 26 | - patch 27 | - delete 28 | - apiGroups: 29 | - apps.foundationdb.org 30 | resources: 31 | - foundationdbclusters 32 | - foundationdbbackups 33 | - foundationdbrestores 34 | verbs: 35 | - get 36 | - list 37 | - watch 38 | - create 39 | - update 40 | - patch 41 | - delete 42 | - apiGroups: 43 | - apps.foundationdb.org 44 | resources: 45 | - foundationdbclusters/status 46 | - foundationdbbackups/status 47 | - foundationdbrestores/status 48 | verbs: 49 | - get 50 | - update 51 | - patch 52 | - apiGroups: 53 | - admissionregistration.k8s.io 54 | resources: 55 | - mutatingwebhookconfigurations 56 | - validatingwebhookconfigurations 57 | verbs: 58 | - get 59 | - list 60 | - watch 61 | - create 62 | - update 63 | - patch 64 | - delete 65 | - apiGroups: 66 | - "" 67 | resources: 68 | - secrets 69 | verbs: 70 | - get 71 | - list 72 | - watch 73 | - create 74 | - update 75 | - patch 76 | - delete 77 | - apiGroups: 78 | - "" 79 | resources: 80 | - services 81 | verbs: 82 | - get 83 | - list 84 | - watch 85 | - create 86 | - update 87 | - patch 88 | - delete 89 | - apiGroups: 90 | - apps 91 | resources: 92 | - deployments 93 | verbs: 94 | - get 95 | - list 96 | - watch 97 | - create 98 | - update 99 | - patch 100 | - delete 101 | - apiGroups: 102 | - coordination.k8s.io 103 | resources: 104 | - leases 105 | verbs: 106 | - get 107 | - list 108 | - watch 109 | - create 110 | - update 111 | - patch 112 | - delete 113 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/rbac/rbac_role_binding.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | {{- if .Values.globalMode.enabled }} 4 | kind: ClusterRoleBinding 5 | {{- else }} 6 | kind: RoleBinding 7 | {{- end }} 8 | metadata: 9 | name: {{ include "fdb-operator.fullname" . }} 10 | labels: 11 | {{- include "fdb-operator.labels" . | nindent 4 }} 12 | roleRef: 13 | apiGroup: rbac.authorization.k8s.io 14 | {{- if .Values.globalMode.enabled }} 15 | kind: ClusterRole 16 | {{- else }} 17 | kind: Role 18 | {{- end }} 19 | name: {{ include "fdb-operator.fullname" . }} 20 | subjects: 21 | - kind: ServiceAccount 22 | name: {{ include "fdb-operator.serviceAccountName" . }} 23 | {{- if .Values.globalMode.enabled }} 24 | namespace: {{ .Release.Namespace }} 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/rbac/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{- if .Values.serviceAccount.create }} 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{ include "fdb-operator.serviceAccountName" . }} 7 | labels: 8 | {{- include "fdb-operator.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- with .Values.serviceAccount.imagePullSecrets }} 14 | imagePullSecrets: 15 | {{- toYaml . | nindent 2 }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/templates/tests/test-connection.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepflowio/deepflow-charts/95c26ff391f36643a9ebb8c35684e6927dacc11f/charts/deepflow/charts/byconity/charts/fdb-operator/templates/tests/test-connection.yaml -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/fdb-operator/values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | image: 3 | repository: foundationdb/fdb-kubernetes-operator 4 | tag: v1.9.0 5 | pullPolicy: IfNotPresent 6 | 7 | initContainerImage: 8 | repository: foundationdb/foundationdb-kubernetes-sidecar 9 | initContainers: 10 | 6.2: 11 | image: 12 | repository: foundationdb/foundationdb-kubernetes-sidecar 13 | tag: 6.2.30-1 14 | pullPolicy: IfNotPresent 15 | 6.3: 16 | image: 17 | repository: foundationdb/foundationdb-kubernetes-sidecar 18 | tag: 6.3.23-1 19 | pullPolicy: IfNotPresent 20 | 7.1: 21 | image: 22 | repository: foundationdb/foundationdb-kubernetes-sidecar 23 | tag: 7.1.15-1 24 | pullPolicy: IfNotPresent 25 | 26 | 27 | globalMode: 28 | enabled: false 29 | 30 | replicas: null 31 | 32 | imagePullSecrets: [] 33 | 34 | annotations: {} 35 | podAnnotations: {} 36 | podLabels: {} 37 | 38 | serviceAccount: 39 | create: true 40 | name: null 41 | imagePullSecrets: [] 42 | annotations: {} 43 | 44 | priorityClassName: null 45 | 46 | securityContext: 47 | runAsUser: 4059 48 | runAsGroup: 4059 49 | fsGroup: 4059 50 | 51 | containerSecurityContext: 52 | allowPrivilegeEscalation: false 53 | privileged: false 54 | capabilities: 55 | drop: 56 | - all 57 | readOnlyRootFilesystem: true 58 | 59 | nodeSelector: {} 60 | 61 | affinity: {} 62 | 63 | tolerations: {} 64 | 65 | resources: 66 | limits: 67 | cpu: 500m 68 | memory: 256Mi 69 | requests: 70 | cpu: 500m 71 | memory: 256Mi 72 | 73 | initContainersResources: 74 | limits: 75 | cpu: 10m 76 | memory: 50Mi 77 | requests: 78 | cpu: 10m 79 | memory: 50Mi 80 | 81 | initContainerSecurityContext: 82 | allowPrivilegeEscalation: false 83 | privileged: false 84 | capabilities: 85 | drop: 86 | - all 87 | readOnlyRootFilesystem: true 88 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/.helmignore: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Patterns to ignore when building packages. 16 | # This supports shell glob matching, relative path matching, and 17 | # negation (prefixed with !). Only one pattern per line. 18 | .DS_Store 19 | # Common VCS dirs 20 | .git/ 21 | .gitignore 22 | .bzr/ 23 | .bzrignore 24 | .hg/ 25 | .hgignore 26 | .svn/ 27 | # Common backup files 28 | *.swp 29 | *.bak 30 | *.tmp 31 | *.orig 32 | *~ 33 | # Various IDEs 34 | .project 35 | .idea/ 36 | *.tmproj 37 | .vscode/ 38 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | name: hdfs 16 | description: Deploys the Hadoop Distributed File System (HDFS) 17 | type: application 18 | version: 1.0.0 # managed version 19 | appVersion: 3.2.2 # managed version 20 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/README.md: -------------------------------------------------------------------------------- 1 | # HDFS Helm Chart 2 | 3 | The Hadoop Distributed File System (HDFS) is a distributed file system designed to run on commodity hardware. This Helm Chart can be used to deploy a HDFS instance onto a Kubernetes cluster. 4 | 5 | By default, this chart deploys: 6 | * a single name node, configured with 1 x 10GB data volume 7 | * 3 x data nodes, each configured with 2 x 10GB data volumes 8 | * configuration for a replication factor of 3 9 | 10 | 11 | ## Deployment 12 | 13 | There are guides for deploying this chart on: 14 | * a local Kubernetes cluster, [using kind (Kubernetes IN Docker)](docs/kind-deployment.md) 15 | * an [AWS EKS cluster](docs/aws-eks-deployment.md) 16 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/docs/aws-eks-deployment.md: -------------------------------------------------------------------------------- 1 | # Deploying HDFS on AWS EKS 2 | All scripts listed here are intended to be run from the kubernetes/hdfs folder 3 | 4 | First follow the [instructions here](../../docs/aws-eks-deployment.md) to provision and configure an [AWS EKS](https://aws.amazon.com/eks/) cluster that the HDFS Helm Chart can be deployed on. 5 | 6 | ## Using ECR 7 | If you are hosting the container images in your AWS account, using ECR, then run the following commands to configure the Helm Chart to use them: 8 | 9 | ```bash 10 | ACCOUNT=$(aws sts get-caller-identity --query Account --output text) 11 | [ "${REGION}" = "" ] && REGION=$(aws configure get region) 12 | [ "${REGION}" = "" ] && REGION=$(curl --silent -m 5 http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | cut -d'"' -f 4) 13 | if [ "${REGION}" = "" ]; then 14 | echo "Unable to detect AWS region, please set \$REGION" 15 | else 16 | REPO_PREFIX="${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com/gchq" 17 | 18 | EXTRA_HELM_ARGS="" 19 | EXTRA_HELM_ARGS+="--set namenode.repository=${REPO_PREFIX}/hdfs " 20 | EXTRA_HELM_ARGS+="--set datanode.repository=${REPO_PREFIX}/hdfs " 21 | EXTRA_HELM_ARGS+="--set shell.repository=${REPO_PREFIX}/hdfs " 22 | fi 23 | ``` 24 | 25 | ## Deploy Helm chart 26 | 27 | ```bash 28 | export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1} 29 | 30 | helm install hdfs . -f ./values-eks-alb.yaml \ 31 | ${EXTRA_HELM_ARGS} \ 32 | --set hdfs.namenode.tag=${HADOOP_VERSION} \ 33 | --set hdfs.datanode.tag=${HADOOP_VERSION} \ 34 | --set hdfs.shell.tag=${HADOOP_VERSION} 35 | 36 | helm test hdfs 37 | ``` 38 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/docs/kind-deployment.md: -------------------------------------------------------------------------------- 1 | # Deploying HDFS using kind 2 | All scripts listed here are intended to be run from the kubernetes/hdfs folder 3 | 4 | First follow the [instructions here](../../docs/kind-deployment.md) to provision and configure a local Kubernetes cluster, using [kind](https://kind.sigs.k8s.io/) (Kubernetes IN Docker), that the HDFS Helm Chart can be deployed on. 5 | 6 | ## Deploying Helm charts 7 | 8 | ```bash 9 | export HADOOP_VERSION=${HADOOP_VERSION:-3.2.1} 10 | 11 | helm install hdfs . \ 12 | --set hdfs.namenode.tag=${HADOOP_VERSION} \ 13 | --set hdfs.datanode.tag=${HADOOP_VERSION} \ 14 | --set hdfs.shell.tag=${HADOOP_VERSION} 15 | 16 | helm test hdfs 17 | ``` 18 | 19 | ## Accessing Web UI (via `kubectl port-forward`) 20 | 21 | ``` 22 | kubectl port-forward svc/hdfs-namenodes 9870:9870 23 | ``` 24 | 25 | Then browse to: http://localhost:9870 26 | 27 | 28 | ## Accessing Web UI (via [Nginx Ingress Controller](https://github.com/kubernetes/ingress-nginx)) 29 | 30 | Register the FQDNs for each component in DNS e.g. 31 | ```bash 32 | echo "127.0.0.1 hdfs.k8s.local" | sudo tee -a /etc/hosts 33 | ``` 34 | 35 | Update the HDFS deployment to route ingress based on FQDNs: 36 | ```bash 37 | helm upgrade hdfs . -f ./values-host-based-ingress.yaml --reuse-values 38 | ``` 39 | 40 | Set up port forwarding to the nginx ingress controller: 41 | ```bash 42 | sudo KUBECONFIG=$HOME/.kube/config kubectl port-forward -n ingress-nginx svc/ingress-nginx 80:80 43 | ``` 44 | 45 | Then browse to: http://hdfs.k8s.local 46 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "hdfs.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "hdfs.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "hdfs.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "hdfs.labels" -}} 38 | helm.sh/chart: {{ include "hdfs.chart" . }} 39 | {{ include "hdfs.selectorLabels" . }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- if .Values.labels }} 45 | {{ toYaml .Values.labels }} 46 | {{- end -}} 47 | {{- end -}} 48 | 49 | {{/* 50 | Selector labels 51 | */}} 52 | {{- define "hdfs.selectorLabels" -}} 53 | app.kubernetes.io/name: {{ include "hdfs.name" . }} 54 | app.kubernetes.io/instance: {{ .Release.Name }} 55 | {{- end -}} 56 | 57 | {{- define "hdfs.namenode.volumes" -}} 58 | {{- $count := int . -}} 59 | {{- range $k, $v := until $count -}} 60 | {{- if gt $v 0 -}},{{- end -}} 61 | /data{{- $v -}}/dfs/name 62 | {{- end -}} 63 | {{- end -}} 64 | 65 | {{- define "hdfs.datanode.volumes" -}} 66 | {{- $count := int . -}} 67 | {{- range $k, $v := until $count -}} 68 | {{- if gt $v 0 -}},{{- end -}} 69 | /data{{- $v -}}/dfs/data 70 | {{- end -}} 71 | {{- end -}} 72 | 73 | {{- define "hdfs.namenodeServiceAccountName" -}} 74 | {{- default (printf "%s-namenode" (include "hdfs.fullname" .)) .Values.config.rackAwareness.serviceAccountName -}} 75 | {{- end -}} 76 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/datanodes-netpol.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | {{- if .Values.networkPolicy.enabled }} 16 | apiVersion: networking.k8s.io/v1 17 | kind: NetworkPolicy 18 | metadata: 19 | name: {{ template "hdfs.fullname" . }}-datanodes 20 | labels: 21 | {{ include "hdfs.labels" . | nindent 4 }} 22 | app.kubernetes.io/component: datanode 23 | spec: 24 | podSelector: 25 | matchLabels: 26 | {{- include "hdfs.selectorLabels" . | nindent 6 }} 27 | app.kubernetes.io/component: datanode 28 | policyTypes: 29 | - Ingress 30 | ingress: 31 | # Allow clients to access client RPC and HTTP servers 32 | - ports: 33 | - port: http 34 | - port: https 35 | - port: data-xfer 36 | # Allow NameNode and DataNodes to access IPC Server 37 | - from: 38 | - podSelector: 39 | matchLabels: 40 | {{- include "hdfs.selectorLabels" . | nindent 10 }} 41 | ports: 42 | - port: ipc 43 | {{- end }} 44 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/datanodes-service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: {{ template "hdfs.fullname" . }}-datanodes 19 | labels: 20 | {{ include "hdfs.labels" . | nindent 4 }} 21 | app.kubernetes.io/component: datanode 22 | spec: 23 | type: ClusterIP 24 | selector: 25 | {{- include "hdfs.selectorLabels" . | nindent 4 }} 26 | app.kubernetes.io/component: datanode 27 | ports: 28 | - name: http 29 | protocol: TCP 30 | port: 80 31 | targetPort: http 32 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/datanodes.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: {{ template "hdfs.fullname" . }}-datanode 19 | labels: 20 | {{- include "hdfs.labels" . | nindent 4 }} 21 | app.kubernetes.io/component: datanode 22 | spec: 23 | podManagementPolicy: Parallel 24 | replicas: {{ .Values.datanode.nodeCount }} 25 | serviceName: {{ template "hdfs.fullname" . }}-datanodes 26 | selector: 27 | matchLabels: 28 | {{- include "hdfs.selectorLabels" . | nindent 6 }} 29 | app.kubernetes.io/component: datanode 30 | volumeClaimTemplates: 31 | {{- range $k, $v := until (int .Values.datanode.dataVolumes.count) }} 32 | - metadata: 33 | name: data{{$v}} 34 | labels: 35 | {{- include "hdfs.labels" $ | nindent 8 }} 36 | spec: 37 | {{- toYaml $.Values.datanode.dataVolumes.pvcTemplateSpec | nindent 6 }} 38 | {{- end }} 39 | template: 40 | metadata: 41 | annotations: 42 | checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} 43 | checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} 44 | labels: 45 | {{- include "hdfs.labels" . | nindent 8 }} 46 | app.kubernetes.io/component: datanode 47 | spec: 48 | initContainers: 49 | - name: wait-for-namenode-startup 50 | image: {{ .Values.alpine.repository}}:{{ .Values.alpine.tag }} 51 | imagePullPolicy: {{ .Values.alpine.imagePullPolicy }} 52 | command: [ 53 | "/bin/sh", 54 | "-c", 55 | 'for i in $(seq 1 300); do nc -z -w3 {{ template "hdfs.fullname" . }}-namenode-0.{{ template "hdfs.fullname" . }}-namenodes:{{ .Values.namenode.ports.serviceRpc }} && exit 0 || sleep 1; done; exit 1' 56 | ] 57 | containers: 58 | - name: datanode 59 | image: {{ .Values.datanode.repository }}:{{ .Values.datanode.tag }} 60 | imagePullPolicy: {{ .Values.datanode.imagePullPolicy }} 61 | args: 62 | - datanode 63 | env: 64 | - name: HADOOP_CONF_DIR 65 | value: {{ .Values.config.path }} 66 | {{- range $key, $value := .Values.datanode.extraEnvVars }} 67 | - name: {{ $key | upper | replace "." "_" }} 68 | value: {{ $value | quote }} 69 | {{- end }} 70 | volumeMounts: 71 | - name: config 72 | mountPath: {{ .Values.config.path }} 73 | readOnly: true 74 | - name: secrets 75 | mountPath: {{ .Values.secrets.path }} 76 | readOnly: true 77 | {{- range $k, $v := until (int .Values.datanode.dataVolumes.count) }} 78 | - name: data{{$v}} 79 | mountPath: /data{{$v}} 80 | {{- end }} 81 | ports: 82 | - name: http 83 | containerPort: {{ .Values.datanode.ports.http }} 84 | - name: https 85 | containerPort: {{ .Values.datanode.ports.https }} 86 | - name: data-xfer 87 | containerPort: {{ .Values.datanode.ports.data }} 88 | - name: ipc 89 | containerPort: {{ .Values.datanode.ports.ipc }} 90 | livenessProbe: 91 | httpGet: 92 | scheme: HTTP 93 | port: http 94 | path: / 95 | initialDelaySeconds: 30 96 | readinessProbe: 97 | httpGet: 98 | scheme: HTTP 99 | port: http 100 | path: / 101 | initialDelaySeconds: 30 102 | resources: 103 | {{- toYaml .Values.datanode.resources | nindent 10 }} 104 | volumes: 105 | - name: config 106 | configMap: 107 | name: {{ template "hdfs.fullname" . }} 108 | optional: false 109 | - name: secrets 110 | secret: 111 | secretName: {{ template "hdfs.fullname" . }} 112 | optional: false 113 | {{- with .Values.datanode.securityContext }} 114 | securityContext: 115 | {{- toYaml . | nindent 8 }} 116 | {{- end }} 117 | {{- with .Values.datanode.affinity }} 118 | affinity: 119 | {{- toYaml . | nindent 8 }} 120 | {{- end }} 121 | {{- with .Values.datanode.nodeSelector }} 122 | nodeSelector: 123 | {{- toYaml . | nindent 8 }} 124 | {{- end }} 125 | {{- with .Values.datanode.tolerations }} 126 | tolerations: 127 | {{- toYaml . | nindent 8 }} 128 | {{- end }} 129 | {{- with .Values.datanode.imagePullSecrets }} 130 | imagePullSecrets: 131 | {{- range . }} 132 | - name: {{ . }} 133 | {{- end }} 134 | {{- end }} 135 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if .Values.ingress.enabled -}} 19 | {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} 20 | apiVersion: networking.k8s.io/v1 21 | {{- else }} 22 | apiVersion: networking.k8s.io/v1beta1 23 | {{- end }} 24 | kind: Ingress 25 | metadata: 26 | name: {{ template "hdfs.fullname" . }} 27 | labels: 28 | {{ include "hdfs.labels" . | nindent 4 }} 29 | app.kubernetes.io/component: namenode 30 | {{- with .Values.ingress.annotations }} 31 | annotations: 32 | {{- toYaml . | nindent 4 }} 33 | {{- end }} 34 | spec: 35 | rules: 36 | - http: 37 | paths: 38 | - path: {{ required "Ingress path required, please set: .Values.ingress.pathPrefix" .Values.ingress.pathPrefix }} 39 | {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} 40 | pathType: Prefix 41 | backend: 42 | service: 43 | name: {{ template "hdfs.fullname" . }}-namenodes 44 | port: 45 | number: {{ .Values.namenode.ports.http }} 46 | {{- else }} 47 | backend: 48 | serviceName: {{ template "hdfs.fullname" . }}-namenodes 49 | servicePort: {{ .Values.namenode.ports.http }} 50 | {{- end }} 51 | 52 | {{- if ne .Values.ingress.host "" }} 53 | host: {{ .Values.ingress.host }} 54 | {{- end }} 55 | {{- end }} 56 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/namenode-netpol.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | {{- if .Values.networkPolicy.enabled }} 16 | apiVersion: networking.k8s.io/v1 17 | kind: NetworkPolicy 18 | metadata: 19 | name: {{ template "hdfs.fullname" . }}-namenodes 20 | labels: 21 | {{ include "hdfs.labels" . | nindent 4 }} 22 | app.kubernetes.io/component: namenode 23 | spec: 24 | podSelector: 25 | matchLabels: 26 | {{- include "hdfs.selectorLabels" . | nindent 6 }} 27 | app.kubernetes.io/component: namenode 28 | policyTypes: 29 | - Ingress 30 | ingress: 31 | # Allow clients to access client RPC and HTTP servers 32 | - ports: 33 | - port: http 34 | - port: https 35 | - port: client-rpc 36 | # Allow NameNode and DataNodes to access Service and Lifeline RPC Servers 37 | - from: 38 | - podSelector: 39 | matchLabels: 40 | {{- include "hdfs.selectorLabels" . | nindent 10 }} 41 | ports: 42 | - port: service-rpc 43 | - port: lifeline-rpc 44 | {{- end }} 45 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/namenode-service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: {{ template "hdfs.fullname" . }}-namenodes 19 | labels: 20 | {{ include "hdfs.labels" . | nindent 4 }} 21 | app.kubernetes.io/component: namenode 22 | spec: 23 | type: ClusterIP 24 | clusterIP: None 25 | selector: 26 | {{- include "hdfs.selectorLabels" . | nindent 4 }} 27 | app.kubernetes.io/component: namenode 28 | ports: 29 | - name: http 30 | protocol: TCP 31 | port: {{ .Values.namenode.ports.http }} 32 | targetPort: http 33 | - name: client-rpc 34 | protocol: TCP 35 | port: {{ .Values.namenode.ports.clientRpc }} 36 | targetPort: client-rpc 37 | - name: service-rpc 38 | protocol: TCP 39 | port: {{ .Values.namenode.ports.serviceRpc }} 40 | targetPort: service-rpc 41 | - name: lifeline-rpc 42 | protocol: TCP 43 | port: {{ .Values.namenode.ports.lifelineRpc }} 44 | targetPort: lifeline-rpc 45 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/namenode-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if and .Values.config.rackAwareness .Values.config.rackAwareness.nodeTopologyLabel (not .Values.config.rackAwareness.serviceAccountName) -}} 19 | apiVersion: v1 20 | kind: ServiceAccount 21 | metadata: 22 | name: {{ template "hdfs.namenodeServiceAccountName" . }} 23 | labels: 24 | {{- include "hdfs.labels" . | nindent 4 }} 25 | app.kubernetes.io/component: namenode 26 | {{- end -}} 27 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/post-install-cmds.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if .Values.postInstallCommands }} 19 | apiVersion: v1 20 | kind: Pod 21 | metadata: 22 | name: {{ template "hdfs.fullname" . }}-post-install-cmds 23 | annotations: 24 | helm.sh/hook: post-install,post-upgrade 25 | helm.sh/hook-delete-policy: hook-succeeded 26 | helm.sh/hook-weight: "10" 27 | labels: 28 | {{- include "hdfs.labels" . | nindent 4 }} 29 | app.kubernetes.io/component: hook 30 | spec: 31 | restartPolicy: OnFailure 32 | containers: 33 | - name: cmds 34 | image: {{ .Values.shell.repository }}:{{ .Values.shell.tag }} 35 | imagePullPolicy: {{ .Values.shell.imagePullPolicy }} 36 | command: ["/bin/bash", "-c", "until hadoop fs -ls / >/dev/null 2>&1 || [ $(( ATTEMPTS++ )) -gt 300 ]; do echo \"$(date) - Waiting for HDFS instance to be ready...\" && sleep 10; done && {{ join " && " .Values.postInstallCommands}}"] 37 | env: 38 | - name: HADOOP_CONF_DIR 39 | value: {{ .Values.config.path }} 40 | volumeMounts: 41 | - name: config 42 | mountPath: {{ .Values.config.path }} 43 | readOnly: true 44 | volumes: 45 | - name: config 46 | configMap: 47 | name: {{ template "hdfs.fullname" . }} 48 | optional: false 49 | {{- with .Values.shell.imagePullSecrets }} 50 | imagePullSecrets: 51 | {{- range . }} 52 | - name: {{ . }} 53 | {{- end }} 54 | {{- end }} 55 | {{- end }} 56 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/rack-awareness-rbac.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if and .Values.config.rackAwareness .Values.config.rackAwareness.nodeTopologyLabel -}} 19 | apiVersion: rbac.authorization.k8s.io/v1 20 | kind: ClusterRole 21 | metadata: 22 | name: {{ template "hdfs.fullname" . }}-namenode 23 | labels: 24 | {{- include "hdfs.labels" . | nindent 4 }} 25 | app.kubernetes.io/component: namenode 26 | rules: 27 | - apiGroups: [""] 28 | resources: ["nodes"] 29 | verbs: ["get"] 30 | --- 31 | apiVersion: rbac.authorization.k8s.io/v1 32 | kind: Role 33 | metadata: 34 | name: {{ template "hdfs.fullname" . }}-namenode 35 | labels: 36 | {{- include "hdfs.labels" . | nindent 4 }} 37 | app.kubernetes.io/component: namenode 38 | rules: 39 | - apiGroups: [""] 40 | resources: ["pods"] 41 | verbs: ["list"] 42 | --- 43 | apiVersion: rbac.authorization.k8s.io/v1 44 | kind: ClusterRoleBinding 45 | metadata: 46 | name: {{ template "hdfs.fullname" . }}-namenode 47 | labels: 48 | {{- include "hdfs.labels" . | nindent 4 }} 49 | app.kubernetes.io/component: namenode 50 | roleRef: 51 | apiGroup: rbac.authorization.k8s.io 52 | kind: ClusterRole 53 | name: {{ template "hdfs.fullname" . }}-namenode 54 | subjects: 55 | - kind: ServiceAccount 56 | name: {{ template "hdfs.namenodeServiceAccountName" . }} 57 | namespace: {{ .Release.Namespace }} 58 | --- 59 | apiVersion: rbac.authorization.k8s.io/v1 60 | kind: RoleBinding 61 | metadata: 62 | name: {{ template "hdfs.fullname" . }}-namenode 63 | labels: 64 | {{- include "hdfs.labels" . | nindent 4 }} 65 | app.kubernetes.io/component: namenode 66 | roleRef: 67 | apiGroup: rbac.authorization.k8s.io 68 | kind: Role 69 | name: {{ template "hdfs.fullname" . }}-namenode 70 | subjects: 71 | - kind: ServiceAccount 72 | name: {{ template "hdfs.namenodeServiceAccountName" . }} 73 | namespace: {{ .Release.Namespace }} 74 | {{- end -}} 75 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/rack-awareness-script.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if and .Values.config.rackAwareness .Values.config.rackAwareness.nodeTopologyLabel -}} 19 | apiVersion: v1 20 | kind: ConfigMap 21 | metadata: 22 | name: {{ template "hdfs.fullname" . }}-rack-awareness 23 | labels: 24 | {{- include "hdfs.labels" . | nindent 4 }} 25 | data: 26 | resolve-rack.sh: | 27 | #!/bin/bash 28 | POD_IP=$1 29 | 30 | API_SERVER="kubernetes.default.svc" 31 | TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) 32 | NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) 33 | CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" 34 | 35 | POD_RESPONSE=$(curl --cacert $CA_CERT --header "Authorization: Bearer $TOKEN" https://$API_SERVER/api/v1/namespaces/$NAMESPACE/pods?fieldSelector=status.podIP%3D$POD_IP) 36 | NODE_NAME=$(echo $POD_RESPONSE | jq -r '.items[0] | .spec.nodeName') 37 | 38 | NODE_RESPONSE=$(curl --cacert $CA_CERT --header "Authorization: Bearer $TOKEN" https://$API_SERVER/api/v1/nodes/$NODE_NAME) 39 | NODE_ZONE=$(echo $NODE_RESPONSE | jq -r '.metadata.labels."{{ .Values.config.rackAwareness.nodeTopologyLabel }}"') 40 | 41 | if [ -z "$NODE_ZONE" ] || [[ "$NODE_ZONE" == "null" ]]; then 42 | NODE_ZONE="default-rack" 43 | fi 44 | 45 | echo "/$NODE_ZONE" 46 | {{- end -}} 47 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Secret 17 | metadata: 18 | name: {{ template "hdfs.fullname" . }} 19 | labels: 20 | {{ include "hdfs.labels" . | indent 4 }} 21 | data: 22 | {{- range $fileName, $secret := .Values.secrets.files }} 23 | {{ $fileName }}: {{ $secret | b64enc | quote }} 24 | {{- end }} 25 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/shell.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if .Values.shell.enabled }} 19 | apiVersion: apps/v1 20 | kind: Deployment 21 | metadata: 22 | name: {{ template "hdfs.fullname" . }}-shell 23 | labels: 24 | {{- include "hdfs.labels" . | nindent 4 }} 25 | app.kubernetes.io/component: shell 26 | spec: 27 | replicas: 1 28 | selector: 29 | matchLabels: 30 | {{- include "hdfs.selectorLabels" . | nindent 6 }} 31 | app.kubernetes.io/component: shell 32 | template: 33 | metadata: 34 | annotations: 35 | checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} 36 | checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} 37 | labels: 38 | {{- include "hdfs.labels" . | nindent 8 }} 39 | app.kubernetes.io/component: shell 40 | spec: 41 | containers: 42 | - name: shell 43 | image: {{ .Values.shell.repository }}:{{ .Values.shell.tag }} 44 | imagePullPolicy: {{ .Values.shell.imagePullPolicy }} 45 | command: 46 | - /bin/bash 47 | stdin: true 48 | tty: true 49 | env: 50 | - name: HADOOP_CONF_DIR 51 | value: {{ .Values.config.path }} 52 | volumeMounts: 53 | - name: config 54 | mountPath: {{ .Values.config.path }} 55 | readOnly: true 56 | - name: secrets 57 | mountPath: {{ .Values.secrets.path }} 58 | readOnly: true 59 | resources: 60 | {{- toYaml .Values.shell.resources | nindent 10 }} 61 | volumes: 62 | - name: config 63 | configMap: 64 | name: {{ template "hdfs.fullname" . }} 65 | optional: false 66 | - name: secrets 67 | secret: 68 | secretName: {{ template "hdfs.fullname" . }} 69 | optional: false 70 | {{- with .Values.shell.affinity }} 71 | affinity: 72 | {{- toYaml . | nindent 8 }} 73 | {{- end }} 74 | {{- with .Values.shell.nodeSelector }} 75 | nodeSelector: 76 | {{- toYaml . | nindent 8 }} 77 | {{- end }} 78 | {{- with .Values.shell.tolerations }} 79 | tolerations: 80 | {{- toYaml . | nindent 8 }} 81 | {{- end }} 82 | {{- with .Values.shell.imagePullSecrets }} 83 | imagePullSecrets: 84 | {{- range . }} 85 | - name: {{ . }} 86 | {{- end }} 87 | {{- end }} 88 | {{- end }} 89 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/tests/auth-test.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Pod 17 | metadata: 18 | name: {{ template "hdfs.fullname" . }}-auth-test 19 | annotations: 20 | helm.sh/hook: test 21 | helm.sh/hook-delete-policy: hook-succeeded 22 | labels: 23 | {{- include "hdfs.labels" . | nindent 4 }} 24 | app.kubernetes.io/component: test 25 | spec: 26 | restartPolicy: Never 27 | containers: 28 | - name: test 29 | image: {{ .Values.shell.repository }}:{{ .Values.shell.tag }} 30 | imagePullPolicy: {{ .Values.shell.imagePullPolicy }} 31 | command: ["/bin/sh", "-c", "hadoop fs -put -f /opt/hadoop/LICENSE.txt / && hadoop fs -ls / && hadoop fs -rm /LICENSE.txt; rc=$?; echo $rc; [ $rc {{ if hasKey .Values.config.hdfsSite "hadoop.security.forward.authentication.url" }}!={{ else }}={{ end }} 0 ]"] 32 | env: 33 | - name: HADOOP_CONF_DIR 34 | value: {{ .Values.config.path }} 35 | volumeMounts: 36 | - name: config 37 | mountPath: {{ .Values.config.path }} 38 | readOnly: true 39 | volumes: 40 | - name: config 41 | configMap: 42 | name: {{ template "hdfs.fullname" . }} 43 | optional: false 44 | {{- with .Values.shell.imagePullSecrets }} 45 | imagePullSecrets: 46 | {{- range . }} 47 | - name: {{ . }} 48 | {{- end }} 49 | {{- end }} 50 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/tests/rack-awareness-test.yaml: -------------------------------------------------------------------------------- 1 | {{- /* 2 | 3 | Copyright 2020 Crown Copyright 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | */ -}} 18 | {{- if and .Values.config.rackAwareness .Values.config.rackAwareness.nodeTopologyLabel -}} 19 | apiVersion: v1 20 | kind: Pod 21 | metadata: 22 | name: {{ template "hdfs.fullname" . }}-rack-awareness-test 23 | annotations: 24 | helm.sh/hook: test 25 | helm.sh/hook-delete-policy: hook-succeeded 26 | labels: 27 | {{- include "hdfs.labels" . | nindent 4 }} 28 | app.kubernetes.io/component: test 29 | spec: 30 | restartPolicy: Never 31 | containers: 32 | - name: test 33 | image: {{ .Values.shell.repository }}:{{ .Values.shell.tag }} 34 | imagePullPolicy: {{ .Values.shell.imagePullPolicy }} 35 | command: ["/bin/sh", "-c", "hdfs dfsadmin -printTopology | grep Rack: | grep -v /default-rack"] 36 | env: 37 | - name: HADOOP_CONF_DIR 38 | value: {{ .Values.config.path }} 39 | volumeMounts: 40 | - name: config 41 | mountPath: {{ .Values.config.path }} 42 | readOnly: true 43 | volumes: 44 | - name: config 45 | configMap: 46 | name: {{ template "hdfs.fullname" . }} 47 | optional: false 48 | {{- with .Values.shell.imagePullSecrets }} 49 | imagePullSecrets: 50 | {{- range . }} 51 | - name: {{ . }} 52 | {{- end }} 53 | {{- end }} 54 | {{- end -}} 55 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/templates/tests/webhdfs-test.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Pod 17 | metadata: 18 | name: {{ template "hdfs.fullname" . }}-webhdfs-test 19 | annotations: 20 | helm.sh/hook: test 21 | helm.sh/hook-delete-policy: hook-succeeded 22 | labels: 23 | {{- include "hdfs.labels" . | nindent 4 }} 24 | app.kubernetes.io/component: test 25 | spec: 26 | restartPolicy: Never 27 | containers: 28 | - name: test 29 | image: {{ .Values.alpine.repository}}:{{ .Values.alpine.tag }} 30 | imagePullPolicy: {{ .Values.alpine.imagePullPolicy }} 31 | command: ["/bin/sh", "-c", "apk update && apk add curl && statusCode=$(curl -f -s -o /dev/null -w \"%{http_code}\" http://{{ template "hdfs.fullname" . }}-namenodes:{{ .Values.namenode.ports.http }}/webhdfs/v1?op=LISTSTATUS) && echo \"${statusCode}\" && [ \"${statusCode}\" {{ if hasKey .Values.config.hdfsSite "dfs.web.authentication.type" }}!={{ else }}={{ end }} \"200\" ]"] 32 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/values-eks-alb.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ingress: 16 | annotations: 17 | kubernetes.io/ingress.class: alb 18 | alb.ingress.kubernetes.io/target-type: ip 19 | alb.ingress.kubernetes.io/scheme: internet-facing 20 | pathPrefix: /* 21 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/values-host-based-ingress.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ingress: 16 | annotations: ~ 17 | host: hdfs.k8s.local 18 | pathPrefix: / 19 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/charts/hdfs/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Crown Copyright 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | alpine: 15 | repository: alpine #if your machine is amd64.please use amd64/alpine 16 | tag: 3.10.2 17 | imagePullPolicy: IfNotPresent 18 | namenode: 19 | repository: gchq/hdfs 20 | tag: 3.2.2 # managed version 21 | imagePullPolicy: IfNotPresent 22 | imagePullSecrets: [] 23 | resources: {} 24 | nodeSelector: {} 25 | tolerations: [] 26 | affinity: {} 27 | securityContext: 28 | runAsNonRoot: true 29 | runAsUser: 1000 30 | runAsGroup: 1000 31 | fsGroup: 1000 32 | ports: 33 | http: 9870 34 | https: 9871 35 | clientRpc: 8020 36 | serviceRpc: 8021 37 | lifelineRpc: 8022 38 | dataVolumes: 39 | count: 1 40 | pvcTemplateSpec: 41 | storageClassName: null 42 | accessModes: 43 | - ReadWriteOnce 44 | resources: 45 | requests: 46 | storage: 10Gi 47 | extraEnvVars: {} 48 | # Example usage: 49 | # HADOOP_HEAPSIZE_MIN: 128m 50 | # hadoop.heapsize.min: 128m 51 | datanode: 52 | nodeCount: 3 53 | repository: gchq/hdfs 54 | tag: 3.2.2 # managed version 55 | imagePullPolicy: IfNotPresent 56 | imagePullSecrets: [] 57 | resources: {} 58 | nodeSelector: {} 59 | tolerations: [] 60 | affinity: {} 61 | securityContext: 62 | runAsNonRoot: true 63 | runAsUser: 1000 64 | runAsGroup: 1000 65 | fsGroup: 1000 66 | ports: 67 | http: 9864 68 | https: 9865 69 | data: 9866 70 | ipc: 9867 71 | dataVolumes: 72 | count: 1 73 | pvcTemplateSpec: 74 | storageClassName: null 75 | accessModes: 76 | - ReadWriteOnce 77 | resources: 78 | requests: 79 | storage: 10Gi 80 | extraEnvVars: {} 81 | # Example usage: 82 | # HADOOP_HEAPSIZE_MIN: 128m 83 | # hadoop.heapsize.min: 128m 84 | shell: 85 | enabled: true 86 | repository: gchq/hdfs 87 | tag: 3.2.2 # managed version 88 | imagePullPolicy: IfNotPresent 89 | imagePullSecrets: [] 90 | resources: {} 91 | nodeSelector: {} 92 | tolerations: [] 93 | affinity: {} 94 | config: 95 | path: /etc/hadoop/conf 96 | coreSite: {} 97 | hdfsSite: 98 | dfs.replication: 3 99 | hadoop.security.authorization: true 100 | rackAwareness: 101 | serviceAccountName: 102 | nodeTopologyLabel: topology.kubernetes.io/zone 103 | secrets: 104 | path: /etc/hadoop/secrets 105 | files: {} 106 | nameOverride: "" 107 | fullnameOverride: "" 108 | labels: {} 109 | ingress: 110 | enabled: true 111 | annotations: 112 | traefik.ingress.kubernetes.io/rule-type: PathPrefixStrip 113 | host: "" 114 | # For Traefik: 115 | # pathPrefix: /hdfs/ 116 | # For Nginx: 117 | pathPrefix: "/hdfs/(.*)" 118 | postInstallCommands: [] 119 | # - hadoop fs -mkdir -p /accumulo 120 | # - hadoop fs -chown accumulo /accumulo 121 | # - hadoop fs -chmod 700 /accumulo 122 | # - hadoop fs -ls / 123 | 124 | networkPolicy: 125 | enabled: true 126 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/cnch-config.yaml: -------------------------------------------------------------------------------- 1 | 2 | catalog: 3 | name_space: default 4 | cnch_type: server 5 | service_discovery: 6 | mode: dns 7 | cluster: default 8 | server: 9 | service: xxx 10 | headless_service: xxx 11 | vw: {} 12 | daemon_manager: 13 | service: xxx 14 | headless_service: xxx 15 | service_discovery_kv: 16 | server_manager: 17 | host_path: data.cnch.server-election 18 | refresh_interval_ms: 1000 19 | expired_interval_ms: 6000 20 | resource_manager: 21 | host_path: data.cnch.resource_manager-election 22 | refresh_interval_ms: 1000 23 | expired_interval_ms: 6000 24 | tso: 25 | host_path: data.cnch.tso-election 26 | refresh_interval_ms: 1000 27 | expired_interval_ms: 6000 28 | tso_service: 29 | tso_window_ms: 3000 30 | tso_get_leader_info_interval_ms: 0 31 | tso_max_retry_count: 3 32 | catalog_service: 33 | type: fdb 34 | fdb: 35 | cluster_file: /etc/byconity/fdb/cluster-file 36 | tcp_port: 9000 37 | http_port: 8123 38 | rpc_port: 8124 39 | exchange_port: 9410 40 | exchange_status_port: 9510 41 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/daemon-manager.yaml: -------------------------------------------------------------------------------- 1 | logger: 2 | level: trace 3 | log: /var/log/byconity/daemon-manager.log 4 | errorlog: /var/log/byconity/daemon-manager.err.log 5 | testlog: /var/log/byconity/daemon-manager.test.log 6 | size: 1000M 7 | count: 10 8 | console: true 9 | listen_host: "::" 10 | path: /var/byconity 11 | tmp_path: /var/byconity/tmp/ 12 | catalog_service: 13 | type: fdb 14 | fdb: 15 | cluster_file: /etc/byconity/fdb/cluster-file 16 | daemon_manager: 17 | port: 9000 18 | http: 19 | port: 8123 # todo remove in the future 20 | receive_timeout: 1800 21 | send_timeout: 1800 22 | workload_thread_interval_ms: 1000 23 | daemon_jobs: 24 | job: 25 | - name: PART_GC 26 | interval: 10000 27 | - name: PART_MERGE 28 | interval: 10000 29 | - name: CONSUMER 30 | interval: 10000 31 | - name: GLOBAL_GC 32 | interval: 5000 33 | - name: TXN_GC 34 | interval: 600000 35 | 36 | hdfs3_config: /etc/byconity/hdfs3.xml 37 | cnch_config: /etc/byconity/cnch-config.yaml 38 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/resource-manager.yaml: -------------------------------------------------------------------------------- 1 | logger: 2 | level: trace 3 | log: /var/log/byconity/resource-manager.log 4 | errorlog: /var/log/byconity/resource-manager.err.log 5 | testlog: /var/log/byconity/resource-manager.test.log 6 | size: 1000M 7 | count: 10 8 | console: true 9 | listen_host: "::" 10 | catalog_service: 11 | type: fdb 12 | fdb: 13 | cluster_file: /etc/byconity/fdb/cluster-file 14 | resource_manager: 15 | port: 9000 16 | http: 17 | port: 8123 18 | receive_timeout: 1800 19 | send_timeout: 1800 20 | vws: 21 | vw: 22 | - name: vw_default 23 | type: default 24 | num_workers: 1 25 | worker_groups: "" 26 | - name: vw_write 27 | type: write 28 | num_workers: 1 29 | worker_groups: "" 30 | cnch_config: /etc/byconity/cnch-config.yaml 31 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/server.yaml: -------------------------------------------------------------------------------- 1 | logger: 2 | level: debug 3 | log: /var/log/byconity/byconity.log 4 | errorlog: /var/log/byconity/byconity.err.log 5 | testlog: /var/log/byconity/byconity.test.log 6 | size: 5000M 7 | count: 50 8 | console: true 9 | listen_host: 10 | - '::' 11 | - '0.0.0.0' 12 | listen_try: 1 13 | 14 | tcp_port: 9000 15 | http_port: 8123 16 | rpc_port: 8124 17 | exchange_port: 9410 18 | exchange_status_port: 9510 19 | 20 | prometheus: 21 | endpoint: /metrics 22 | port: 0 23 | metrics: true 24 | events: true 25 | asynchronous_metrics: true 26 | part_metrics: false 27 | path: /var/byconity 28 | tmp_path: /var/byconity/tmp 29 | users_config: /etc/byconity/users.yaml 30 | dictionaries_config: '*_dictionary.xml' 31 | format_schema_path: /var/byconity/format_schemas/ 32 | cnch_type: server 33 | tso_service: 34 | tso_window_ms: 3000 35 | tso_get_leader_info_interval_ms: 0 36 | tso_max_retry_count: 3 37 | catalog_service: 38 | type: fdb 39 | fdb: 40 | cluster_file: /etc/byconity/fdb/cluster-file 41 | 42 | hdfs3_config: /etc/byconity/hdfs3.xml 43 | cnch_kafka_log: 44 | database: cnch_system 45 | table: cnch_kafka_log 46 | flush_max_row_count: 100000 47 | flush_interval_milliseconds: 60000 48 | disk_cache_strategies: 49 | simple: 50 | lru_max_size: 2199023255552 # 2Ti 51 | merge_selector: dance # todo remove later when engine support 52 | plan_segment_log: 53 | database: system 54 | table: plan_segment_log 55 | partition_by: event_date 56 | flush_interval_milliseconds: 15000 57 | query_log: 58 | database: system 59 | table: query_log 60 | partition_by: event_date 61 | flush_interval_milliseconds: 15000 62 | mark_cache_size: 5368709120 63 | #zookeeper: {} 64 | cnch_config: /etc/byconity/cnch-config.yaml 65 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/tso.yaml: -------------------------------------------------------------------------------- 1 | logger: 2 | level: trace 3 | log: /var/log/byconity/tso.log 4 | errorlog: /var/log/byconity/tso.err.log 5 | testlog: /var/log/byconity/tso.test.log 6 | size: 1000M 7 | count: 10 8 | console: true 9 | listen_host: "::" 10 | path: /var/byconity/tso 11 | tmp_path: /var/byconity/tmp 12 | tso_service: 13 | type: fdb 14 | fdb: 15 | cluster_file: /etc/byconity/fdb/cluster-file 16 | port: 9000 17 | http: 18 | port: 8123 19 | receive_timeout: 1800 20 | send_timeout: 1800 21 | tso_window_ms: 3000 22 | tso_get_leader_info_interval_ms: 0 23 | # keeper_server: 24 | # tcp_port: 9181 # tcp port, Receive request (like Zookeeper) 25 | # server_id: 0 # should be uniq, used in leader-election TODO 26 | # log_storage_path: /var/byconity/tso/log 27 | # snapshot_storage_path: /var/byconity/tso/snapshots 28 | # coordination_settings: 29 | # operation_timeout_ms: 10000 30 | # session_timeout_ms: 30000 31 | # raft_logs_level: warning 32 | # compress_logs: 1 33 | # zookeeper: {} 34 | cnch_config: /etc/byconity/cnch-config.yaml 35 | catalog_service: 36 | type: fdb 37 | fdb: 38 | cluster_file: /etc/byconity/fdb/cluster-file 39 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/users.yaml: -------------------------------------------------------------------------------- 1 | profiles: 2 | default: 3 | load_balancing: random 4 | log_queries: 1 5 | max_execution_time: 180 6 | exchange_timeout_ms: 300000 7 | 8 | users: 9 | default: 10 | networks: 11 | ip: ::/0 12 | password: "" 13 | profile: default 14 | quota: default 15 | server: 16 | networks: 17 | ip: ::/0 18 | password: "" 19 | profile: default 20 | quota: default 21 | probe: 22 | networks: 23 | ip: ::/0 24 | password: "" 25 | profile: default 26 | quota: default 27 | 28 | quotas: 29 | default: 30 | interval: 31 | duration: 3600 32 | queries: 0 33 | errors: 0 34 | result_rows: 0 35 | read_rows: 0 36 | execution_time: 0 37 | 38 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/files/worker.yaml: -------------------------------------------------------------------------------- 1 | logger: 2 | level: debug 3 | log: /var/log/byconity/byconity.log 4 | errorlog: /var/log/byconity/byconity.err.log 5 | testlog: /var/log/byconity/byconity.test.log 6 | size: 5000M 7 | count: 50 8 | console: true 9 | listen_host: 10 | - '::' 11 | - '0.0.0.0' 12 | listen_try: 1 13 | 14 | tcp_port: 9000 15 | http_port: 8123 16 | rpc_port: 8124 17 | exchange_port: 9410 18 | exchange_status_port: 9510 19 | 20 | prometheus: 21 | endpoint: /metrics 22 | port: 0 23 | metrics: true 24 | events: true 25 | asynchronous_metrics: true 26 | part_metrics: false 27 | path: /var/byconity 28 | tmp_path: /var/byconity/tmp 29 | users_config: /etc/byconity/users.yaml 30 | dictionaries_config: '*_dictionary.xml' 31 | format_schema_path: /var/byconity/format_schemas/ 32 | cnch_type: worker 33 | tso_service: 34 | tso_window_ms: 3000 35 | tso_get_leader_info_interval_ms: 0 36 | tso_max_retry_count: 3 37 | catalog_service: 38 | type: fdb 39 | fdb: 40 | cluster_file: /etc/byconity/fdb/cluster-file 41 | 42 | hdfs3_config: /etc/byconity/hdfs3.xml 43 | cnch_kafka_log: 44 | database: cnch_system 45 | table: cnch_kafka_log 46 | flush_max_row_count: 100000 47 | flush_interval_milliseconds: 60000 48 | disk_cache_strategies: 49 | simple: 50 | lru_max_size: 2199023255552 # 2Ti 51 | merge_selector: dance # todo remove later when engine support 52 | plan_segment_log: 53 | database: system 54 | table: plan_segment_log 55 | partition_by: event_date 56 | flush_interval_milliseconds: 15000 57 | query_log: 58 | database: system 59 | table: query_log 60 | partition_by: event_date 61 | flush_interval_milliseconds: 15000 62 | mark_cache_size: 5368709120 63 | cnch_config: /etc/byconity/cnch-config.yaml 64 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | Byconity cluster "{{ include "byconity.fullname" . }}" deployed successfully! Please follow instructions to access your cluster. 2 | 3 | $ kubectl -n {{ .Release.Namespace }} exec -it sts/{{ include "byconity.fullname" . }}-server -- bash 4 | $ clickhouse client 5 | 172.16.1.1 :) show databases 6 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "byconity.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "byconity.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "byconity.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "byconity.labels" -}} 37 | helm.sh/chart: {{ include "byconity.chart" . }} 38 | {{ include "byconity.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "byconity.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "byconity.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "byconity.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "byconity.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | 64 | {{/* 65 | Create common environment variables to use 66 | */}} 67 | {{- define "byconity.commonEnvs" -}} 68 | - name: MY_POD_NAMESPACE 69 | valueFrom: 70 | fieldRef: 71 | fieldPath: "metadata.namespace" 72 | - name: MY_POD_NAME 73 | valueFrom: 74 | fieldRef: 75 | fieldPath: "metadata.name" 76 | - name: MY_UID 77 | valueFrom: 78 | fieldRef: 79 | apiVersion: v1 80 | fieldPath: "metadata.uid" 81 | - name: MY_POD_IP 82 | valueFrom: 83 | fieldRef: 84 | fieldPath: "status.podIP" 85 | - name: MY_HOST_IP 86 | valueFrom: 87 | fieldRef: 88 | fieldPath: "status.hostIP" 89 | {{- end }} 90 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/daemonmanager-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-dm-run 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: daemon-manager 8 | data: 9 | run: | 10 | #!/bin/bash 11 | exec "${APP_ROOT}/bin/daemon-manager" --config "/etc/byconity/daemon-manager.yaml" 12 | 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: {{ include "byconity.fullname" . }}-dm-config 18 | labels: 19 | {{- include "byconity.labels" . | nindent 4 }} 20 | byconity-role: daemon-manager 21 | data: 22 | daemon-manager.yaml: | 23 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "server") "psm" (printf "%s-%s" (include "byconity.fullname" .) "server") }} 24 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 25 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service) }} 26 | {{- .Files.Get "files/daemon-manager.yaml" | fromYaml | merge .Values.byconity.daemonManager.configOverwrite $.Values.byconity.configOverwrite $service_discovery | toYaml | nindent 4 }} 27 | hdfs3.xml: | 28 | 29 | {{- range $key, $val := $.Values.byconity.hdfs3Config }} 30 | 31 | {{ $key }} 32 | {{ $val }} 33 | 34 | {{- end }} 35 | 36 | cnch-config.yaml: | 37 | {{- $ports := dict "tcp_port" .Values.byconity.ports.tcp "http_port" .Values.byconity.ports.http "rpc_port" .Values.byconity.ports.rpc "exchange_port" .Values.byconity.ports.exchange "exchange_status_port" .Values.byconity.ports.exchangeStatus }} 38 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "server") "psm" (printf "%s-%s" (include "byconity.fullname" .) "server") }} 39 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 40 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") }} 41 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") }} 42 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 43 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" .)) }} 44 | {{- $hdfs_addr_dict := dict "hdfs_addr" .Values.byconity.hdfs_address }} 45 | {{- $hdfs_ha_nameservice_dict := dict "hdfs_ha_nameservice" .Values.byconity.hdfs_ha_nameservice }} 46 | {{- $hdfs_addr := (empty .Values.byconity.hdfs_ha_nameservice | ternary $hdfs_addr_dict $hdfs_ha_nameservice_dict) }} 47 | {{- .Files.Get "files/cnch-config.yaml" | fromYaml | merge $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 48 | 49 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/daemonmanager-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-daemon-manager 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: daemon-manager 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | {{- include "byconity.selectorLabels" . | nindent 4 }} 12 | byconity-role: daemon-manager 13 | ports: 14 | - name: port0 15 | port: 9000 16 | targetPort: tcp 17 | 18 | --- 19 | apiVersion: v1 20 | kind: Service 21 | metadata: 22 | name: {{ include "byconity.fullname" . }}-daemon-manager-headless 23 | labels: 24 | {{- include "byconity.labels" . | nindent 4 }} 25 | byconity-role: daemon-manager 26 | spec: 27 | type: ClusterIP 28 | clusterIP: None 29 | selector: 30 | {{- include "byconity.selectorLabels" . | nindent 4 }} 31 | byconity-role: daemon-manager 32 | ports: 33 | - name: port0 34 | port: 9000 35 | targetPort: tcp 36 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/daemonmanager.yaml: -------------------------------------------------------------------------------- 1 | {{- $imageRepository := tpl $.Values.image.repository . }} 2 | {{- with .Values.byconity.daemonManager }} 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: {{ include "byconity.fullname" $ }}-daemon-manager 7 | labels: 8 | {{- include "byconity.labels" $ | nindent 4 }} 9 | byconity-role: daemon-manager 10 | spec: 11 | replicas: {{ .replicas }} 12 | selector: 13 | matchLabels: 14 | {{- include "byconity.selectorLabels" $ | nindent 6 }} 15 | byconity-role: daemon-manager 16 | strategy: 17 | type: Recreate 18 | template: 19 | metadata: 20 | {{- with .podAnnotations }} 21 | annotations: 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | labels: 25 | {{- include "byconity.selectorLabels" $ | nindent 8 }} 26 | byconity-role: daemon-manager 27 | spec: 28 | containers: 29 | - name: byconity-daemon-manager 30 | image: "{{ $imageRepository }}:{{ $.Values.image.tag }}" 31 | imagePullPolicy: {{ $.Values.image.imagePullPolicy }} 32 | env: 33 | {{- toYaml $.Values.byconity.commonEnvs | nindent 12 }} 34 | - name: BYCONITY_ROLE 35 | value: daemon-manager 36 | {{- if $.Values.byconity.additionalEnvs }} 37 | {{- toYaml $.Values.byconity.additionalEnvs | nindent 12 }} 38 | {{- end }} 39 | {{- if .resources }} 40 | resources: 41 | {{- toYaml .resources | nindent 12 }} 42 | {{- end }} 43 | ports: 44 | - name: tcp 45 | containerPort: {{ $.Values.byconity.ports.tcp }} 46 | readinessProbe: 47 | {{- .readinessProbe | toYaml | nindent 12 }} 48 | livenessProbe: 49 | {{ .livenessProbe | toYaml | nindent 12 }} 50 | lifecycle: 51 | postStart: 52 | exec: 53 | command: ["/opt/byconity/scripts/lifecycle/poststart"] 54 | preStop: 55 | exec: 56 | command: ["/opt/byconity/scripts/lifecycle/prestop"] 57 | volumeMounts: 58 | - name: run 59 | mountPath: /etc/service/byconity/run 60 | subPath: run 61 | - name: config 62 | mountPath: /etc/byconity 63 | - name: lifecycle 64 | mountPath: /opt/byconity/scripts/lifecycle 65 | - name: fdb-config 66 | mountPath: /etc/byconity/fdb 67 | {{- if $.Values.byconity.additionalVolumes.volumeMounts }} 68 | {{- toYaml $.Values.byconity.additionalVolumes.volumeMounts | nindent 12 }} 69 | {{- end }} 70 | {{- if .securityContext }} 71 | securityContext: 72 | {{- toYaml .securityContext | nindent 12 }} 73 | {{- end }} 74 | volumes: 75 | - name: run 76 | configMap: 77 | name: {{ include "byconity.fullname" $ }}-dm-run 78 | defaultMode: 0755 79 | - name: config 80 | configMap: 81 | name: {{ include "byconity.fullname" $ }}-dm-config 82 | - name: lifecycle 83 | configMap: 84 | name: {{ include "byconity.fullname" $ }}-lifecycle 85 | defaultMode: 0755 86 | - name: fdb-config 87 | configMap: 88 | name: {{ include "byconity.fullname" $ }}-fdb-config 89 | {{- if $.Values.byconity.additionalVolumes.volumes }} 90 | {{- toYaml $.Values.byconity.additionalVolumes.volumes | nindent 8 }} 91 | {{- end }} 92 | hostNetwork: {{ .hostNetwork }} 93 | {{- with .nodeSelector }} 94 | nodeSelector: 95 | {{- toYaml . | nindent 8 }} 96 | {{- end }} 97 | {{- with .affinity }} 98 | affinity: 99 | {{- toYaml . | nindent 8 }} 100 | {{- end }} 101 | {{- with .tolerations }} 102 | tolerations: 103 | {{- toYaml . | nindent 8 }} 104 | {{- end }} 105 | imagePullSecrets: 106 | {{- with $.Values.global.imagePullSecrets }} 107 | {{- toYaml . | nindent 8 }} 108 | {{- end }} 109 | {{- with $.Values.imagePullSecrets }} 110 | {{- toYaml . | nindent 8 }} 111 | {{- end }} 112 | {{- end }} 113 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/fdb-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.byconity.use_existing_fdb }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "byconity.fullname" $ }}-fdb-config 6 | data: 7 | cluster-file: | 8 | {{ $.Values.byconity.fdb_cluster_file }} 9 | {{ end }} 10 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/fdb_shell.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.fdb.enableCliPod .Values.fdb.enabled }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "byconity.fullname" . }}-fdbcli 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app.kubernetes.io/name: fdbcli 11 | app.kubernetes.io/instance: {{ include "byconity.fullname" . }} 12 | app.kubernetes.io/component: cli 13 | template: 14 | metadata: 15 | labels: 16 | app.kubernetes.io/name: fdbcli 17 | app.kubernetes.io/instance: {{ include "byconity.fullname" . }} 18 | app.kubernetes.io/component: cli 19 | spec: 20 | imagePullSecrets: 21 | {{- with .Values.global.imagePullSecrets }} 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | {{- with .Values.imagePullSecrets }} 25 | {{- toYaml . | nindent 8 }} 26 | {{- end }} 27 | containers: 28 | - name: fdbcli 29 | image: "{{ tpl $.Values.fdbShell.image.repository . }}/foundationdb:{{ .Values.fdb.version }}" 30 | env: 31 | - name: FDB_CLUSTER_FILE 32 | value: /mnt/config-volume/cluster-file 33 | volumeMounts: 34 | - name: config-volume 35 | mountPath: /mnt/config-volume 36 | resources: 37 | limits: 38 | cpu: "1" 39 | memory: 1Gi 40 | requests: 41 | cpu: "1" 42 | memory: 1Gi 43 | {{- with .Values.byconity.server.affinity }} 44 | affinity: 45 | {{- toYaml . | nindent 8 }} 46 | {{- end }} 47 | volumes: 48 | - name: config-volume 49 | configMap: 50 | name: {{ include "byconity.fullname" . }}-fdb-config 51 | {{- end }} 52 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.byconity.ingress.enabled -}} 2 | {{- $fullName := include "byconity.fullname" . -}} 3 | {{- $svcPort := 8123 -}} 4 | {{- if and .Values.byconity.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.byconity.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.byconity.ingress.annotations "kubernetes.io/ingress.class" .Values.byconity.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | labels: 20 | {{- include "byconity.labels" . | nindent 4 }} 21 | {{- with .Values.byconity.ingress.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | spec: 26 | {{- if and .Values.byconity.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 27 | ingressClassName: {{ .Values.byconity.ingress.className }} 28 | {{- end }} 29 | {{- if .Values.byconity.ingress.tls }} 30 | tls: 31 | {{- range .Values.byconity.ingress.tls }} 32 | - hosts: 33 | {{- range .hosts }} 34 | - {{ . | quote }} 35 | {{- end }} 36 | secretName: {{ .secretName }} 37 | {{- end }} 38 | {{- end }} 39 | rules: 40 | {{- range .Values.byconity.ingress.hosts }} 41 | - host: {{ .host | quote }} 42 | http: 43 | paths: 44 | {{- range .paths }} 45 | - path: {{ .path }} 46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} 47 | pathType: {{ .pathType }} 48 | {{- end }} 49 | backend: 50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 51 | service: 52 | name: {{ $fullName }}-server 53 | port: 54 | number: {{ $svcPort }} 55 | {{- else }} 56 | serviceName: {{ $fullName }}-server 57 | servicePort: {{ $svcPort }} 58 | {{- end }} 59 | {{- end }} 60 | {{- end }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/lifecycle.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-lifecycle 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: server 8 | data: 9 | poststart: | 10 | #!/bin/bash 11 | set -eo pipefail 12 | {{- .Values.byconity.postStart | nindent 4}} 13 | prestop: |- 14 | #!/bin/bash 15 | set -eo pipefail 16 | if [[ "${BYCONITY_ROLE}" == "worker" ]]; then 17 | curl --fail -X GET "http://127.0.0.1:{{ .Values.byconity.ports.http }}/api/resource_report?action=stop" 18 | fi 19 | {{- .Values.byconity.preStop | nindent 4 }} 20 | readiness: | 21 | #!/bin/bash 22 | set -euo pipefail 23 | QUERY_TIMEOUT=5 24 | timeout -k 1 "${QUERY_TIMEOUT}" clickhouse-client -h 127.0.0.1 --port "{{ .Values.byconity.ports.tcp }}" --user "probe" --password "{{ .Values.byconity.usersOverwrite.users.probe.password }}" --max_execution_time "$((QUERY_TIMEOUT-1))" -n -q "select 1" 2>&1 25 | {{- .Values.byconity.readiness | nindent 4}} 26 | liveness: | 27 | #!/bin/bash 28 | set -euo pipefail 29 | QUERY_TIMEOUT=5 30 | timeout -k 1 "${QUERY_TIMEOUT}" clickhouse-client -h 127.0.0.1 --port "{{ .Values.byconity.ports.tcp }}" --user "probe" --password "{{ .Values.byconity.usersOverwrite.users.probe.password }}" --max_execution_time "$((QUERY_TIMEOUT-1))" -n -q "select 1" 2>&1 31 | {{- .Values.byconity.readiness | nindent 4}} 32 | readiness-tso: | 33 | #!/bin/bash 34 | set -euo pipefail 35 | curl --connect-timeout 3 --fail http://localhost:9000/health 36 | {{- .Values.byconity.readiness | nindent 4}} 37 | liveness-tso: | 38 | #!/bin/bash 39 | set -euo pipefail 40 | curl --connect-timeout 3 --fail http://localhost:9000/health 41 | {{- .Values.byconity.readiness | nindent 4}} 42 | readiness-dm: | 43 | #!/bin/bash 44 | set -euo pipefail 45 | curl --connect-timeout 3 --fail http://localhost:9000/health 46 | {{- .Values.byconity.readiness | nindent 4}} 47 | liveness-dm: | 48 | #!/bin/bash 49 | set -euo pipefail 50 | curl --connect-timeout 3 --fail http://localhost:9000/health 51 | {{- .Values.byconity.readiness | nindent 4}} 52 | readiness-rm: | 53 | #!/bin/bash 54 | set -euo pipefail 55 | curl --connect-timeout 3 --fail http://localhost:9000/health 56 | {{- .Values.byconity.readiness | nindent 4}} 57 | liveness-rm: | 58 | #!/bin/bash 59 | set -euo pipefail 60 | curl --connect-timeout 3 --fail http://localhost:9000/health 61 | {{- .Values.byconity.readiness | nindent 4}} 62 | {{- range $key, $data := .Values.byconity.lifecycleExtraConfig }} 63 | {{ $key | quote }}: {{ $data | quote }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/resourcemanager-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-rm-run 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: resource-manager 8 | data: 9 | run: | 10 | #!/bin/bash 11 | exec "${APP_ROOT}/bin/resource-manager" --config "/etc/byconity/resource-manager.yaml" 12 | 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: {{ include "byconity.fullname" . }}-rm-config 18 | labels: 19 | {{- include "byconity.labels" . | nindent 4 }} 20 | byconity-role: resource-manager 21 | data: 22 | resource-manager.yaml: | 23 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 24 | {{- $service_discovery := dict "service_discovery" (dict "tso" $tso_service) }} 25 | {{- .Files.Get "files/resource-manager.yaml" | fromYaml | merge .Values.byconity.resourceManager.configOverwrite $service_discovery | toYaml | nindent 4 }} 26 | cnch-config.yaml: | 27 | {{- $ports := dict "tcp_port" .Values.byconity.ports.tcp "http_port" .Values.byconity.ports.http "rpc_port" .Values.byconity.ports.rpc "exchange_port" .Values.byconity.ports.exchange "exchange_status_port" .Values.byconity.ports.exchangeStatus }} 28 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "server") "psm" (printf "%s-%s" (include "byconity.fullname" .) "server") }} 29 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 30 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") }} 31 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") }} 32 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 33 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" .)) }} 34 | {{- $hdfs_addr_dict := dict "hdfs_addr" .Values.byconity.hdfs_address }} 35 | {{- $hdfs_ha_nameservice_dict := dict "hdfs_ha_nameservice" .Values.byconity.hdfs_ha_nameservice }} 36 | {{- $hdfs_addr := (empty .Values.byconity.hdfs_ha_nameservice | ternary $hdfs_addr_dict $hdfs_ha_nameservice_dict) }} 37 | {{- .Files.Get "files/cnch-config.yaml" | fromYaml | merge $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 38 | 39 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/resourcemanager-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-resource-manager 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: resource-manager 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | {{- include "byconity.selectorLabels" . | nindent 4 }} 12 | byconity-role: resource-manager 13 | ports: 14 | - name: port0 15 | port: 9000 16 | targetPort: tcp 17 | 18 | --- 19 | apiVersion: v1 20 | kind: Service 21 | metadata: 22 | name: {{ include "byconity.fullname" . }}-resource-manager-headless 23 | labels: 24 | {{- include "byconity.labels" . | nindent 4 }} 25 | byconity-role: resource-manager 26 | spec: 27 | type: ClusterIP 28 | clusterIP: None 29 | selector: 30 | {{- include "byconity.selectorLabels" . | nindent 4 }} 31 | byconity-role: resource-manager 32 | ports: 33 | - name: port0 34 | port: 9000 35 | targetPort: tcp 36 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/resourcemanager.yaml: -------------------------------------------------------------------------------- 1 | {{- $imageRepository := tpl $.Values.image.repository . }} 2 | {{- with .Values.byconity.resourceManager }} 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: {{ include "byconity.fullname" $ }}-resource-manager 7 | labels: 8 | {{- include "byconity.labels" $ | nindent 4 }} 9 | byconity-role: resource-manager 10 | spec: 11 | replicas: {{ .replicas }} 12 | selector: 13 | matchLabels: 14 | {{- include "byconity.selectorLabels" $ | nindent 6 }} 15 | byconity-role: resource-manager 16 | strategy: 17 | type: Recreate 18 | template: 19 | metadata: 20 | {{- with .podAnnotations }} 21 | annotations: 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | labels: 25 | {{- include "byconity.selectorLabels" $ | nindent 8 }} 26 | byconity-role: resource-manager 27 | spec: 28 | containers: 29 | - name: byconity-resource-manager 30 | image: "{{ $imageRepository }}:{{ $.Values.image.tag }}" 31 | imagePullPolicy: {{ $.Values.image.imagePullPolicy }} 32 | env: 33 | {{- toYaml $.Values.byconity.commonEnvs | nindent 12 }} 34 | - name: BYCONITY_ROLE 35 | value: resource-manager 36 | {{- if $.Values.byconity.additionalEnvs }} 37 | {{- toYaml $.Values.byconity.additionalEnvs | nindent 12 }} 38 | {{- end }} 39 | {{- if .resources }} 40 | resources: 41 | {{- toYaml .resources | nindent 12 }} 42 | {{- end }} 43 | ports: 44 | - name: tcp 45 | containerPort: {{ $.Values.byconity.ports.tcp }} 46 | readinessProbe: 47 | {{- .readinessProbe | toYaml | nindent 12 }} 48 | livenessProbe: 49 | {{- .livenessProbe | toYaml | nindent 12 }} 50 | lifecycle: 51 | postStart: 52 | exec: 53 | command: ["/opt/byconity/scripts/lifecycle/poststart"] 54 | preStop: 55 | exec: 56 | command: ["/opt/byconity/scripts/lifecycle/prestop"] 57 | volumeMounts: 58 | - name: run 59 | mountPath: /etc/service/byconity/run 60 | subPath: run 61 | - name: config 62 | mountPath: /etc/byconity 63 | - name: lifecycle 64 | mountPath: /opt/byconity/scripts/lifecycle 65 | - name: fdb-config 66 | mountPath: /etc/byconity/fdb 67 | {{- if $.Values.byconity.additionalVolumes.volumeMounts }} 68 | {{- toYaml $.Values.byconity.additionalVolumes.volumeMounts | nindent 12 }} 69 | {{- end }} 70 | {{- if .securityContext }} 71 | securityContext: 72 | {{- toYaml .securityContext | nindent 12 }} 73 | {{- end }} 74 | volumes: 75 | - name: run 76 | configMap: 77 | name: {{ include "byconity.fullname" $ }}-rm-run 78 | defaultMode: 0755 79 | - name: config 80 | configMap: 81 | name: {{ include "byconity.fullname" $ }}-rm-config 82 | - name: lifecycle 83 | configMap: 84 | name: {{ include "byconity.fullname" $ }}-lifecycle 85 | defaultMode: 0755 86 | - name: fdb-config 87 | configMap: 88 | name: {{ include "byconity.fullname" $ }}-fdb-config 89 | {{- if $.Values.byconity.additionalVolumes.volumes }} 90 | {{- toYaml $.Values.byconity.additionalVolumes.volumes | nindent 8 }} 91 | {{- end }} 92 | hostNetwork: {{ .hostNetwork }} 93 | {{- with .nodeSelector }} 94 | nodeSelector: 95 | {{- toYaml . | nindent 8 }} 96 | {{- end }} 97 | {{- with .affinity }} 98 | affinity: 99 | {{- toYaml . | nindent 8 }} 100 | {{- end }} 101 | {{- with .tolerations }} 102 | tolerations: 103 | {{- toYaml . | nindent 8 }} 104 | {{- end }} 105 | imagePullSecrets: 106 | {{- with $.Values.global.imagePullSecrets }} 107 | {{- toYaml . | nindent 8 }} 108 | {{- end }} 109 | {{- with $.Values.imagePullSecrets }} 110 | {{- toYaml . | nindent 8 }} 111 | {{- end }} 112 | {{- end }} 113 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/server-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-server-run 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: server 8 | data: 9 | run: | 10 | #!/bin/bash 11 | chown root. /var/byconity /var/log/byconity 12 | exec "${APP_ROOT}/bin/clickhouse-server" --config "/etc/byconity/server.yaml" 13 | 14 | --- 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: {{ include "byconity.fullname" . }}-server-config 19 | labels: 20 | {{- include "byconity.labels" . | nindent 4 }} 21 | byconity-role: server 22 | data: 23 | server.yaml: | 24 | {{- $ports := dict "tcp_port" .Values.byconity.ports.tcp "http_port" .Values.byconity.ports.http "rpc_port" .Values.byconity.ports.rpc "exchange_port" .Values.byconity.ports.exchange "exchange_status_port" .Values.byconity.ports.exchangeStatus }} 25 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "server") "psm" (printf "%s-%s" (include "byconity.fullname" .) "server") }} 26 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 27 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") }} 28 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") }} 29 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 30 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" .)) }} 31 | {{- $hdfs_addr := dict "hdfs_addr" .Values.byconity.hdfs_address }} 32 | {{- .Files.Get "files/server.yaml" | fromYaml | merge .Values.byconity.server.configOverwrite .Values.byconity.configOverwrite $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 33 | users.yaml: | 34 | {{- .Files.Get "files/users.yaml" | fromYaml | merge .Values.byconity.usersOverwrite | toYaml | nindent 4 }} 35 | hdfs3.xml: | 36 | 37 | {{- range $key, $val := $.Values.byconity.hdfs3Config }} 38 | 39 | {{ $key }} 40 | {{ $val }} 41 | 42 | {{- end }} 43 | 44 | cnch-config.yaml: | 45 | {{- $ports := dict "tcp_port" .Values.byconity.ports.tcp "http_port" .Values.byconity.ports.http "rpc_port" .Values.byconity.ports.rpc "exchange_port" .Values.byconity.ports.exchange "exchange_status_port" .Values.byconity.ports.exchangeStatus }} 46 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "server") "psm" (printf "%s-%s" (include "byconity.fullname" .) "server") }} 47 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 48 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") }} 49 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") }} 50 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 51 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" .)) }} 52 | {{- $hdfs_addr_dict := dict "hdfs_addr" .Values.byconity.hdfs_address }} 53 | {{- $hdfs_ha_nameservice_dict := dict "hdfs_ha_nameservice" .Values.byconity.hdfs_ha_nameservice }} 54 | {{- $hdfs_addr := (empty .Values.byconity.hdfs_ha_nameservice | ternary $hdfs_addr_dict $hdfs_ha_nameservice_dict) }} 55 | {{- .Files.Get "files/cnch-config.yaml" | fromYaml | merge $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 56 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/server-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-server 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: server 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | {{- include "byconity.selectorLabels" . | nindent 4 }} 12 | byconity-role: server 13 | ports: 14 | - name: port0 15 | port: {{ $.Values.byconity.ports.tcp }} 16 | targetPort: tcp 17 | - name: port1 18 | port: {{ $.Values.byconity.ports.rpc }} 19 | targetPort: rpc 20 | - name: port2 21 | port: {{ $.Values.byconity.ports.http }} 22 | targetPort: http 23 | - name: port3 24 | port: {{ $.Values.byconity.ports.tcpSecure }} 25 | targetPort: tcp-secure 26 | - name: port4 27 | port: {{ $.Values.byconity.ports.https }} 28 | targetPort: https 29 | - name: port5 30 | port: {{ $.Values.byconity.ports.exchange }} 31 | targetPort: exchange 32 | - name: port6 33 | port: {{ $.Values.byconity.ports.exchangeStatus }} 34 | targetPort: exchange-status 35 | 36 | --- 37 | apiVersion: v1 38 | kind: Service 39 | metadata: 40 | name: {{ include "byconity.fullname" . }}-server-headless 41 | labels: 42 | {{- include "byconity.labels" . | nindent 4 }} 43 | byconity-role: server 44 | spec: 45 | type: ClusterIP 46 | clusterIP: None 47 | selector: 48 | {{- include "byconity.selectorLabels" . | nindent 4 }} 49 | byconity-role: server 50 | ports: 51 | - name: port0 52 | port: {{ $.Values.byconity.ports.tcp }} 53 | targetPort: tcp 54 | - name: port1 55 | port: {{ $.Values.byconity.ports.rpc }} 56 | targetPort: rpc 57 | - name: port2 58 | port: {{ $.Values.byconity.ports.http }} 59 | targetPort: http 60 | - name: port3 61 | port: {{ $.Values.byconity.ports.tcpSecure }} 62 | targetPort: tcp-secure 63 | - name: port4 64 | port: {{ $.Values.byconity.ports.https }} 65 | targetPort: https 66 | - name: port5 67 | port: {{ $.Values.byconity.ports.exchange }} 68 | targetPort: exchange 69 | - name: port6 70 | port: {{ $.Values.byconity.ports.exchangeStatus }} 71 | targetPort: exchange-status 72 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/server.yaml: -------------------------------------------------------------------------------- 1 | {{- $imageRepository := tpl $.Values.image.repository . }} 2 | {{- with .Values.byconity.server }} 3 | apiVersion: apps/v1 4 | kind: StatefulSet 5 | metadata: 6 | name: {{ include "byconity.fullname" $ }}-server 7 | labels: 8 | {{- include "byconity.labels" $ | nindent 4 }} 9 | byconity-role: server 10 | spec: 11 | replicas: {{ .replicas }} 12 | serviceName: {{ include "byconity.fullname" $ }}-server 13 | selector: 14 | matchLabels: 15 | {{- include "byconity.selectorLabels" $ | nindent 6 }} 16 | byconity-role: server 17 | updateStrategy: 18 | type: RollingUpdate 19 | template: 20 | metadata: 21 | {{- with .podAnnotations }} 22 | annotations: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | labels: 26 | {{- include "byconity.selectorLabels" $ | nindent 8 }} 27 | byconity-role: server 28 | spec: 29 | containers: 30 | - name: byconity-server 31 | image: "{{ $imageRepository }}:{{ $.Values.image.tag }}" 32 | imagePullPolicy: {{ $.Values.image.imagePullPolicy }} 33 | env: 34 | {{- toYaml $.Values.byconity.commonEnvs | nindent 12 }} 35 | - name: BYCONITY_ROLE 36 | value: server 37 | {{- if $.Values.byconity.additionalEnvs }} 38 | {{- toYaml $.Values.byconity.additionalEnvs | nindent 12 }} 39 | {{- end }} 40 | {{- if .resources }} 41 | resources: 42 | {{- toYaml .resources | nindent 12 }} 43 | {{- end }} 44 | ports: 45 | - name: tcp 46 | containerPort: {{ $.Values.byconity.ports.tcp }} 47 | - name: rpc 48 | containerPort: {{ $.Values.byconity.ports.rpc }} 49 | - name: http 50 | containerPort: {{ $.Values.byconity.ports.http }} 51 | - name: tcp-secure 52 | containerPort: {{ $.Values.byconity.ports.tcpSecure }} 53 | - name: https 54 | containerPort: {{ $.Values.byconity.ports.https }} 55 | - name: exchange 56 | containerPort: {{ $.Values.byconity.ports.exchange }} 57 | - name: exchange-status 58 | containerPort: {{ $.Values.byconity.ports.exchangeStatus }} 59 | readinessProbe: 60 | {{- .readinessProbe | toYaml | nindent 12 }} 61 | livenessProbe: 62 | {{- .livenessProbe | toYaml | nindent 12 }} 63 | lifecycle: 64 | postStart: 65 | exec: 66 | command: ["/opt/byconity/scripts/lifecycle/poststart"] 67 | preStop: 68 | exec: 69 | command: ["/opt/byconity/scripts/lifecycle/prestop"] 70 | volumeMounts: 71 | - name: run 72 | mountPath: /etc/service/byconity/run 73 | subPath: run 74 | - name: config 75 | mountPath: /etc/byconity 76 | - name: lifecycle 77 | mountPath: /opt/byconity/scripts/lifecycle 78 | - name: local-disk 79 | mountPath: /var/byconity 80 | - name: log 81 | mountPath: /var/log/byconity 82 | - name: fdb-config 83 | mountPath: /etc/byconity/fdb 84 | {{- if $.Values.byconity.additionalVolumes.volumeMounts }} 85 | {{- toYaml $.Values.byconity.additionalVolumes.volumeMounts | nindent 12 }} 86 | {{- end }} 87 | {{- if .securityContext }} 88 | securityContext: 89 | {{- toYaml .securityContext | nindent 12 }} 90 | {{- end }} 91 | volumes: 92 | - name: run 93 | configMap: 94 | name: {{ include "byconity.fullname" $ }}-server-run 95 | defaultMode: 0755 96 | - name: config 97 | configMap: 98 | name: {{ include "byconity.fullname" $ }}-server-config 99 | - name: lifecycle 100 | configMap: 101 | name: {{ include "byconity.fullname" $ }}-lifecycle 102 | defaultMode: 0755 103 | - name: fdb-config 104 | configMap: 105 | name: {{ include "byconity.fullname" $ }}-fdb-config 106 | {{- if $.Values.byconity.additionalVolumes.volumes }} 107 | {{- toYaml $.Values.byconity.additionalVolumes.volumes | nindent 8 }} 108 | {{- end }} 109 | hostNetwork: {{ .hostNetwork }} 110 | {{- with .nodeSelector }} 111 | nodeSelector: 112 | {{- toYaml . | nindent 8 }} 113 | {{- end }} 114 | {{- with .affinity }} 115 | affinity: 116 | {{- toYaml . | nindent 8 }} 117 | {{- end }} 118 | {{- with .tolerations }} 119 | tolerations: 120 | {{- toYaml . | nindent 8 }} 121 | {{- end }} 122 | imagePullSecrets: 123 | {{- with $.Values.global.imagePullSecrets }} 124 | {{- toYaml . | nindent 8 }} 125 | {{- end }} 126 | {{- with $.Values.imagePullSecrets }} 127 | {{- toYaml . | nindent 8 }} 128 | {{- end }} 129 | volumeClaimTemplates: 130 | - metadata: 131 | name: local-disk 132 | spec: 133 | {{- toYaml .storage.localDisk.pvcSpec | nindent 8 }} 134 | - metadata: 135 | name: log 136 | spec: 137 | {{- toYaml .storage.log.pvcSpec | nindent 8 }} 138 | {{- end }} 139 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/tso-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-tso-run 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: tso 8 | data: 9 | run: | 10 | #!/bin/bash 11 | exec "${APP_ROOT}/bin/tso-server" --config "/etc/byconity/tso.yaml" 12 | 13 | --- 14 | apiVersion: v1 15 | kind: ConfigMap 16 | metadata: 17 | name: {{ include "byconity.fullname" . }}-tso-config 18 | labels: 19 | {{- include "byconity.labels" . | nindent 4 }} 20 | byconity-role: tso 21 | data: 22 | tso.yaml: | 23 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 24 | {{- $service_discovery := dict "service_discovery" (dict "tso" $tso_service) }} 25 | {{- .Files.Get "files/tso.yaml" | fromYaml | merge .Values.byconity.tso.configOverwrite $service_discovery | toYaml | nindent 4 }} 26 | cnch-config.yaml: | 27 | {{- $ports := dict "tcp_port" .Values.byconity.ports.tcp "http_port" .Values.byconity.ports.http "rpc_port" .Values.byconity.ports.rpc "exchange_port" .Values.byconity.ports.exchange "exchange_status_port" .Values.byconity.ports.exchangeStatus }} 28 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "server") "psm" (printf "%s-%s" (include "byconity.fullname" .) "server") }} 29 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" .) "tso") }} 30 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "daemon-manager") }} 31 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" .) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" .) "resource-manager") }} 32 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 33 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" .)) }} 34 | {{- $hdfs_addr_dict := dict "hdfs_addr" .Values.byconity.hdfs_address }} 35 | {{- $hdfs_ha_nameservice_dict := dict "hdfs_ha_nameservice" .Values.byconity.hdfs_ha_nameservice }} 36 | {{- $hdfs_addr := (empty .Values.byconity.hdfs_ha_nameservice | ternary $hdfs_addr_dict $hdfs_ha_nameservice_dict) }} 37 | {{- .Files.Get "files/cnch-config.yaml" | fromYaml | merge $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 38 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/tso-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-tso 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: tso 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | {{- include "byconity.selectorLabels" . | nindent 4 }} 12 | byconity-role: tso 13 | ports: 14 | - name: port0 15 | port: 9000 16 | targetPort: tcp 17 | - name: port1 18 | port: 9445 # keeper log 19 | targetPort: tcp 20 | - name: port2 21 | port: 9181 # keeper 22 | targetPort: tcp 23 | 24 | --- 25 | apiVersion: v1 26 | kind: Service 27 | metadata: 28 | name: {{ include "byconity.fullname" . }}-tso-headless 29 | labels: 30 | {{- include "byconity.labels" . | nindent 4 }} 31 | byconity-role: tso 32 | spec: 33 | type: ClusterIP 34 | clusterIP: None 35 | selector: 36 | {{- include "byconity.selectorLabels" . | nindent 4 }} 37 | byconity-role: tso 38 | ports: 39 | - name: port0 40 | port: 9000 41 | targetPort: tcp 42 | - name: port1 43 | port: 9445 # keeper log 44 | targetPort: tcp 45 | - name: port2 46 | port: 9181 # keeper 47 | targetPort: tcp 48 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/tso.yaml: -------------------------------------------------------------------------------- 1 | {{- $imageRepository := tpl $.Values.image.repository . }} 2 | {{- with .Values.byconity.tso }} 3 | apiVersion: apps/v1 4 | kind: StatefulSet 5 | metadata: 6 | name: {{ include "byconity.fullname" $ }}-tso 7 | labels: 8 | {{- include "byconity.labels" $ | nindent 4 }} 9 | byconity-role: tso 10 | spec: 11 | replicas: {{ .replicas }} 12 | serviceName: {{ include "byconity.fullname" $ }}-tso 13 | selector: 14 | matchLabels: 15 | {{- include "byconity.selectorLabels" $ | nindent 6 }} 16 | byconity-role: tso 17 | updateStrategy: 18 | type: RollingUpdate 19 | template: 20 | metadata: 21 | {{- with .podAnnotations }} 22 | annotations: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | labels: 26 | {{- include "byconity.selectorLabels" $ | nindent 8 }} 27 | byconity-role: tso 28 | spec: 29 | containers: 30 | - name: byconity-tso 31 | image: "{{ $imageRepository }}:{{ $.Values.image.tag }}" 32 | imagePullPolicy: {{ $.Values.image.imagePullPolicy }} 33 | env: 34 | {{- toYaml $.Values.byconity.commonEnvs | nindent 12 }} 35 | - name: BYCONITY_ROLE 36 | value: tso 37 | {{- if $.Values.byconity.additionalEnvs }} 38 | {{- toYaml $.Values.byconity.additionalEnvs | nindent 12 }} 39 | {{- end }} 40 | - name: TSO_IP 41 | valueFrom: 42 | fieldRef: 43 | fieldPath: "status.podIP" 44 | {{- if .resources }} 45 | resources: 46 | {{- toYaml .resources | nindent 12 }} 47 | {{- end }} 48 | ports: 49 | - name: tcp 50 | containerPort: {{ $.Values.byconity.ports.tcp }} 51 | readinessProbe: 52 | {{- .readinessProbe | toYaml | nindent 12 }} 53 | livenessProbe: 54 | {{- .livenessProbe | toYaml | nindent 12 }} 55 | lifecycle: 56 | postStart: 57 | exec: 58 | command: ["/opt/byconity/scripts/lifecycle/poststart"] 59 | preStop: 60 | exec: 61 | command: ["/opt/byconity/scripts/lifecycle/prestop"] 62 | volumeMounts: 63 | - name: run 64 | mountPath: /etc/service/byconity/run 65 | subPath: run 66 | - name: config 67 | mountPath: /etc/byconity 68 | - name: lifecycle 69 | mountPath: /opt/byconity/scripts/lifecycle 70 | - name: fdb-config 71 | mountPath: /etc/byconity/fdb 72 | - name: local-disk 73 | mountPath: /var/byconity 74 | - name: log 75 | mountPath: /var/log/byconity 76 | {{- if $.Values.byconity.tso.additionalVolumes.volumeMounts }} 77 | {{- toYaml $.Values.byconity.tso.additionalVolumes.volumeMounts | nindent 12 }} 78 | {{- end }} 79 | {{- if .securityContext }} 80 | securityContext: 81 | {{- toYaml .securityContext | nindent 12 }} 82 | {{- end }} 83 | volumes: 84 | - name: run 85 | configMap: 86 | name: {{ include "byconity.fullname" $ }}-tso-run 87 | defaultMode: 0755 88 | - name: config 89 | configMap: 90 | name: {{ include "byconity.fullname" $ }}-tso-config 91 | - name: lifecycle 92 | configMap: 93 | name: {{ include "byconity.fullname" $ }}-lifecycle 94 | defaultMode: 0755 95 | - name: fdb-config 96 | configMap: 97 | name: {{ include "byconity.fullname" $ }}-fdb-config 98 | {{- if $.Values.byconity.tso.additionalVolumes.volumes }} 99 | {{- toYaml $.Values.byconity.tso.additionalVolumes.volumes | nindent 8 }} 100 | {{- end }} 101 | hostNetwork: {{ .hostNetwork }} 102 | {{- with .nodeSelector }} 103 | nodeSelector: 104 | {{- toYaml . | nindent 8 }} 105 | {{- end }} 106 | {{- with .affinity }} 107 | affinity: 108 | {{- toYaml . | nindent 8 }} 109 | {{- end }} 110 | {{- with .tolerations }} 111 | tolerations: 112 | {{- toYaml . | nindent 8 }} 113 | {{- end }} 114 | imagePullSecrets: 115 | {{- with $.Values.global.imagePullSecrets }} 116 | {{- toYaml . | nindent 8 }} 117 | {{- end }} 118 | {{- with $.Values.imagePullSecrets }} 119 | {{- toYaml . | nindent 8 }} 120 | {{- end }} 121 | volumeClaimTemplates: 122 | - metadata: 123 | name: local-disk 124 | spec: 125 | {{- toYaml .storage.localDisk.pvcSpec | nindent 8 }} 126 | - metadata: 127 | name: log 128 | spec: 129 | {{- toYaml .storage.log.pvcSpec | nindent 8 }} 130 | {{- if $.Values.byconity.tso.additionalVolumeClaimTemplates }} 131 | {{- toYaml $.Values.byconity.tso.additionalVolumeClaimTemplates | nindent 4 }} 132 | {{- end }} 133 | {{- end }} 134 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/vw-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "byconity.fullname" . }}-worker-run 5 | labels: 6 | {{- include "byconity.labels" . | nindent 4 }} 7 | byconity-role: worker 8 | data: 9 | run: | 10 | #!/bin/bash 11 | chown root. /var/byconity /var/log/byconity 12 | exec "${APP_ROOT}/bin/clickhouse-server" --config "/etc/byconity/worker.yaml" 13 | 14 | {{- range .Values.byconity.virtualWarehouses }} 15 | --- 16 | apiVersion: v1 17 | kind: ConfigMap 18 | metadata: 19 | name: {{ include "byconity.fullname" $ }}-vw-{{ .name | replace "_" "-" }}-config 20 | labels: 21 | {{- include "byconity.labels" $ | nindent 4 }} 22 | byconity-role: worker 23 | byconity-vw: {{ .name }} 24 | data: 25 | worker.yaml: | 26 | {{- $ports := dict "tcp_port" $.Values.byconity.ports.tcp "http_port" $.Values.byconity.ports.http "rpc_port" $.Values.byconity.ports.rpc "exchange_port" $.Values.byconity.ports.exchange "exchange_status_port" $.Values.byconity.ports.exchangeStatus }} 27 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "server") "psm" (printf "%s-%s" (include "byconity.fullname" $) "server") }} 28 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" $) "tso") }} 29 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" $) "daemon-manager") }} 30 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" $) "resource-manager") }} 31 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 32 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" $)) }} 33 | {{- $hdfs_addr := dict "hdfs_addr" $.Values.byconity.hdfs_address }} 34 | {{- $.Files.Get "files/worker.yaml" | fromYaml | merge .configOverwrite $.Values.byconity.configOverwrite $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 35 | users.yaml: | 36 | {{- $.Files.Get "files/users.yaml" | fromYaml | merge $.Values.byconity.usersOverwrite | toYaml | nindent 4 }} 37 | hdfs3.xml: | 38 | 39 | {{- range $key, $val := $.Values.byconity.hdfs3Config }} 40 | 41 | {{ $key }} 42 | {{ $val }} 43 | 44 | {{- end }} 45 | 46 | cnch-config.yaml: | 47 | {{- $ports := dict "tcp_port" $.Values.byconity.ports.tcp "http_port" $.Values.byconity.ports.http "rpc_port" $.Values.byconity.ports.rpc "exchange_port" $.Values.byconity.ports.exchange "exchange_status_port" $.Values.byconity.ports.exchangeStatus }} 48 | {{- $server_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "server") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "server") "psm" (printf "%s-%s" (include "byconity.fullname" $) "server") }} 49 | {{- $tso_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "tso") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "tso") "psm" (printf "%s-%s" (include "byconity.fullname" $) "tso") }} 50 | {{- $dm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "daemon-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "daemon-manager") "psm" (printf "%s-%s" (include "byconity.fullname" $) "daemon-manager") }} 51 | {{- $rm_service := dict "service" (printf "%s-%s" (include "byconity.fullname" $) "resource-manager") "headless_service" (printf "%s-%s-headless" (include "byconity.fullname" $) "resource-manager") "psm" (printf "%s-%s" (include "byconity.fullname" $) "resource-manager") }} 52 | {{- $service_discovery := dict "service_discovery" (dict "server" $server_service "tso" $tso_service "daemon_manager" $dm_service "resource_manager" $rm_service) }} 53 | {{- $catalog := dict "catalog" (dict "name_space" (include "byconity.fullname" $)) }} 54 | {{- $hdfs_addr_dict := dict "hdfs_addr" $.Values.byconity.hdfs_address }} 55 | {{- $hdfs_ha_nameservice_dict := dict "hdfs_ha_nameservice" $.Values.byconity.hdfs_ha_nameservice }} 56 | {{- $hdfs_addr := (empty $.Values.byconity.hdfs_ha_nameservice | ternary $hdfs_addr_dict $hdfs_ha_nameservice_dict) }} 57 | {{- $.Files.Get "files/cnch-config.yaml" | fromYaml | merge .configOverwrite $hdfs_addr $catalog $service_discovery $ports | toYaml | nindent 4 }} 58 | {{- end }} 59 | -------------------------------------------------------------------------------- /charts/deepflow/charts/byconity/templates/vw-service.yaml: -------------------------------------------------------------------------------- 1 | {{- range .Values.byconity.virtualWarehouses }} 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: {{ include "byconity.fullname" $ }}-vw-{{ .name | replace "_" "-"}} 7 | labels: 8 | {{- include "byconity.labels" $ | nindent 4 }} 9 | byconity-role: worker 10 | byconity-vw: {{ .name }} 11 | spec: 12 | type: ClusterIP 13 | selector: 14 | {{- include "byconity.selectorLabels" $ | nindent 4 }} 15 | byconity-role: worker 16 | ports: 17 | - name: port0 18 | port: {{ $.Values.byconity.ports.tcp }} 19 | targetPort: tcp 20 | - name: port1 21 | port: {{ $.Values.byconity.ports.rpc }} 22 | targetPort: rpc 23 | - name: port2 24 | port: {{ $.Values.byconity.ports.http }} 25 | targetPort: http 26 | - name: port3 27 | port: {{ $.Values.byconity.ports.tcpSecure }} 28 | targetPort: tcp-secure 29 | - name: port4 30 | port: {{ $.Values.byconity.ports.https }} 31 | targetPort: https 32 | - name: port5 33 | port: {{ $.Values.byconity.ports.exchange }} 34 | targetPort: exchange 35 | - name: port6 36 | port: {{ $.Values.byconity.ports.exchangeStatus }} 37 | targetPort: exchange-status 38 | 39 | --- 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | name: {{ include "byconity.fullname" $ }}-vw-{{ .name | replace "_" "-"}}-headless 44 | labels: 45 | {{- include "byconity.labels" $ | nindent 4 }} 46 | byconity-role: worker 47 | byconity-vw: {{ .name }} 48 | spec: 49 | type: ClusterIP 50 | clusterIP: None 51 | selector: 52 | {{- include "byconity.selectorLabels" $ | nindent 4 }} 53 | byconity-role: worker 54 | ports: 55 | - name: port0 56 | port: {{ $.Values.byconity.ports.tcp }} 57 | targetPort: tcp 58 | - name: port1 59 | port: {{ $.Values.byconity.ports.rpc }} 60 | targetPort: rpc 61 | - name: port2 62 | port: {{ $.Values.byconity.ports.http }} 63 | targetPort: http 64 | - name: port3 65 | port: {{ $.Values.byconity.ports.tcpSecure }} 66 | targetPort: tcp-secure 67 | - name: port4 68 | port: {{ $.Values.byconity.ports.https }} 69 | targetPort: https 70 | - name: port5 71 | port: {{ $.Values.byconity.ports.exchange }} 72 | targetPort: exchange 73 | - name: port6 74 | port: {{ $.Values.byconity.ports.exchangeStatus }} 75 | targetPort: exchange-status 76 | {{- end }} 77 | -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: clickhouse 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.000 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "21.8.15.7" 25 | -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "clickhouse.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "clickhouse.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "clickhouse.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "clickhouse.labels" -}} 37 | helm.sh/chart: {{ include "clickhouse.chart" . }} 38 | {{ include "clickhouse.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "clickhouse.selectorLabels" -}} 49 | app: deepflow 50 | component: clickhouse 51 | app.kubernetes.io/name: {{ include "clickhouse.name" . }} 52 | app.kubernetes.io/instance: {{ .Release.Name }} 53 | {{- end }} 54 | 55 | {{/* 56 | Create the name of the service account to use 57 | */}} 58 | {{- define "clickhouse.serviceAccountName" -}} 59 | {{- if .Values.serviceAccount.create }} 60 | {{- default (include "clickhouse.fullname" .) .Values.serviceAccount.name }} 61 | {{- else }} 62 | {{- default "default" .Values.serviceAccount.name }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/templates/_volumeClaimTemplates.tpl: -------------------------------------------------------------------------------- 1 | {{- /* PVC templates */ -}} 2 | {{- define "persistentVolumeClaim" -}} 3 | {{- range $index, $volume := .Values.storageConfig.persistence }} 4 | - kind: PersistentVolumeClaim 5 | apiVersion: v1 6 | metadata: 7 | name: {{ $volume.name }} 8 | annotations: 9 | {{- toYaml $volume.annotations | nindent 8 }} 10 | spec: 11 | accessModes: 12 | {{- toYaml $volume.accessModes | nindent 8 }} 13 | resources: 14 | requests: 15 | storage: {{ $volume.size | quote }} 16 | {{- if (tpl $volume.storageClass $) }} 17 | storageClassName: {{ tpl $volume.storageClass $ | quote }} 18 | {{- end }} 19 | {{- if $volume.selector }} 20 | selector: 21 | {{- toYaml $volume.selector | nindent 8 }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "clickhouse.fullname" . }} 5 | labels: 6 | {{- include "clickhouse.labels" . | nindent 4 }} 7 | {{- if .Values.service.annotations }} 8 | annotations: 9 | {{ toYaml .Values.service.annotations | indent 4 }} 10 | {{- end }} 11 | spec: 12 | {{- if .Values.service.clusterIP }} 13 | clusterIP: {{ .Values.service.clusterIP }} 14 | {{- end }} 15 | {{- if .Values.service.externalIPs }} 16 | externalIPs: 17 | {{ toYaml .Values.service.externalIPs | indent 4 }} 18 | {{- end }} 19 | {{- if .Values.service.loadBalancerIP }} 20 | loadBalancerIP: {{ .Values.service.loadBalancerIP }} 21 | {{- end }} 22 | {{- if .Values.service.loadBalancerSourceRanges }} 23 | loadBalancerSourceRanges: 24 | {{- range $cidr := .Values.service.loadBalancerSourceRanges }} 25 | - {{ $cidr }} 26 | {{- end }} 27 | {{- end }} 28 | {{- if ne .Values.service.type "ClusterIP" }} 29 | externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} 30 | {{- end }} 31 | ports: 32 | {{- $serviceType := .Values.service.type -}} 33 | {{- range .Values.service.ports }} 34 | - name: {{ tpl .name $ }} 35 | port: {{ tpl (toString .port) $ }} 36 | targetPort: {{ tpl (toString .targetPort) $ }} 37 | {{- if and (eq $serviceType "NodePort") (.nodePort) }} 38 | nodePort: {{ tpl (toString .nodePort) $ }} 39 | {{- end }} 40 | protocol: {{ tpl .protocol $ }} 41 | {{- end }} 42 | {{- if .Values.service.additionalPorts }} 43 | {{ toYaml .Values.service.additionalPorts | indent 2 }} 44 | {{- end }} 45 | selector: 46 | {{- include "clickhouse.selectorLabels" . | nindent 4 }} 47 | type: "{{ .Values.service.type }}" 48 | --- 49 | apiVersion: v1 50 | kind: Service 51 | metadata: 52 | name: {{ include "clickhouse.fullname" . }}-headless 53 | labels: 54 | {{- include "clickhouse.labels" . | nindent 4 }} 55 | spec: 56 | clusterIP: None 57 | ports: 58 | {{- range .Values.service.ports }} 59 | - name: {{ tpl .name $ }} 60 | port: {{ tpl (toString .port) $ }} 61 | targetPort: {{ tpl (toString .targetPort) $ }} 62 | protocol: {{ tpl .protocol $ }} 63 | {{- end }} 64 | selector: 65 | {{- include "clickhouse.selectorLabels" . | nindent 4 }} 66 | type: ClusterIP -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/templates/statefulset.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (not $.Values.global.allInOneLocalStorage ) (eq ( tpl $.Values.storageConfig.generateType . ) "hostPath") (not $.Values.nodeAffinityLabelSelector) }} 2 | {{- fail "You must set nodeAffinityLabelSelector" -}} 3 | {{- end}} 4 | apiVersion: apps/v1 5 | kind: StatefulSet 6 | metadata: 7 | name: {{ include "clickhouse.fullname" . }} 8 | labels: 9 | {{- include "clickhouse.labels" . | nindent 4 }} 10 | spec: 11 | replicas: {{ tpl (toString .Values.replicas) . }} 12 | podManagementPolicy: {{ tpl .Values.podManagementPolicy . }} 13 | serviceName: {{ include "clickhouse.fullname" . }}-headless 14 | selector: 15 | matchLabels: 16 | {{- include "clickhouse.selectorLabels" . | nindent 6 }} 17 | template: 18 | metadata: 19 | {{- with .Values.podAnnotations }} 20 | annotations: 21 | {{- toYaml . | nindent 8 }} 22 | {{- end }} 23 | labels: 24 | {{- include "clickhouse.selectorLabels" . | nindent 8 }} 25 | {{- with .Values.podLabels }} 26 | {{- toYaml . | nindent 8 }} 27 | {{- end }} 28 | spec: 29 | hostNetwork: {{ tpl (toString .Values.hostNetwork) . }} 30 | dnsPolicy: {{ tpl .Values.dnsPolicy . }} 31 | imagePullSecrets: 32 | {{- with .Values.global.imagePullSecrets }} 33 | {{- toYaml . | nindent 8 }} 34 | {{- end }} 35 | {{- with .Values.imagePullSecrets }} 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | securityContext: 39 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 40 | {{- if .Values.chmodContainer.enabled }} 41 | initContainers: 42 | - name: clickhouse-init 43 | image: "{{ tpl .Values.image.repository . }}:{{ tpl (toString .Values.image.tag) . }}" 44 | volumeMounts: 45 | - name: clickhouse-path 46 | mountPath: /var/lib/clickhouse/ 47 | - name: clickhouse-storage-path 48 | mountPath: /var/lib/clickhouse_storage/ 49 | command: ['sh', '-c', 'chown clickhouse:clickhouse /var/lib/clickhouse_storage;chown clickhouse:clickhouse /var/lib/clickhouse/'] 50 | {{- end }} 51 | containers: 52 | - name: clickhouse 53 | securityContext: 54 | {{- toYaml .Values.securityContext | nindent 12 }} 55 | image: "{{ tpl .Values.image.repository . }}:{{ tpl (toString .Values.image.tag) . }}" 56 | imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} 57 | env: 58 | - name: TZ 59 | value: "{{ tpl .Values.timezone . }}" 60 | ports: 61 | - name: tcp 62 | containerPort: 9000 63 | protocol: TCP 64 | readinessProbe: 65 | tcpSocket: 66 | port: tcp 67 | initialDelaySeconds: 5 68 | periodSeconds: 10 69 | livenessProbe: 70 | tcpSocket: 71 | port: tcp 72 | initialDelaySeconds: 15 73 | periodSeconds: 20 74 | resources: 75 | {{- toYaml .Values.resources | nindent 12 }} 76 | volumeMounts: 77 | - name: clickhouse-config 78 | mountPath: /etc/clickhouse-server/ 79 | - name: clickhouse-path 80 | mountPath: /var/lib/clickhouse/ 81 | - name: clickhouse-storage-path 82 | mountPath: /var/lib/clickhouse_storage/ 83 | {{- range .Values.storageConfig.extraClickhousePath }} 84 | - name: {{ .volumeName }} 85 | mountPath: {{ .mountPath }} 86 | {{- end }} 87 | {{- with .Values.nodeSelector }} 88 | nodeSelector: 89 | {{- toYaml . | nindent 8 }} 90 | {{- end }} 91 | affinity: 92 | {{- include "nodeaffinity" . | indent 6 }} 93 | {{- include "podAffinity" . | indent 6 }} 94 | {{- include "podAntiAffinity" . | indent 6 }} 95 | {{- if or .Values.global.tolerations .Values.tolerations }} 96 | tolerations: 97 | {{- if .Values.global.tolerations }} 98 | {{- toYaml .Values.global.tolerations | nindent 8 }} 99 | {{- end }} 100 | {{- if .Values.tolerations }} 101 | {{- toYaml .Values.tolerations | nindent 8 }} 102 | {{- end }} 103 | {{- end }} 104 | volumes: 105 | - name: clickhouse-config 106 | configMap: 107 | name: {{ include "clickhouse.fullname" . }}-config 108 | {{- if eq ( tpl .Values.storageConfig.generateType . ) "hostPath" }} 109 | - name: clickhouse-path 110 | hostPath: 111 | path: {{ tpl .Values.storageConfig.hostPath . }}/clickhouse 112 | type: DirectoryOrCreate 113 | - name: clickhouse-storage-path 114 | hostPath: 115 | path: {{ tpl .Values.storageConfig.hostPath . }}/clickhouse-storage 116 | type: DirectoryOrCreate 117 | {{- end }} 118 | {{- if eq ( tpl $.Values.storageConfig.generateType . ) "persistentVolumeClaim" }} 119 | volumeClaimTemplates: 120 | {{- include "persistentVolumeClaim" . | indent 2 }} 121 | {{- end }} 122 | -------------------------------------------------------------------------------- /charts/deepflow/charts/clickhouse/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for clickhouse. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | global: 5 | podAntiAffinityLabelSelector: [] 6 | podAntiAffinityTermLabelSelector: [] 7 | podAffinityLabelSelector: [] 8 | podAffinityTermLabelSelector: [] 9 | nodeAffinityLabelSelector: [] 10 | nodeAffinityTermLabelSelector: [] 11 | password: 12 | clickhouse: 13 | 14 | replicas: "1" 15 | hostNetwork: "false" 16 | dnsPolicy: "ClusterFirst" 17 | podManagementPolicy: "OrderedReady" 18 | image: 19 | repository: clickhouse/clickhouse-server 20 | pullPolicy: IfNotPresent 21 | # Overrides the image tag whose default is the chart appVersion. 22 | tag: 22.8.6.71 23 | 24 | imagePullSecrets: [] 25 | nameOverride: "" 26 | fullnameOverride: "" 27 | 28 | podAnnotations: {} 29 | 30 | podSecurityContext: {} 31 | # fsGroup: 2000 32 | 33 | chmodContainer: 34 | enabled: true 35 | securityContext: {} 36 | # capabilities: 37 | # drop: 38 | # - ALL 39 | # readOnlyRootFilesystem: true 40 | # runAsNonRoot: true 41 | # runAsUser: 1000 42 | timezone: "Asia/Shanghai" 43 | storageConfig: 44 | ## persistentVolumeClaim/hostPath 45 | generateType: "{{ if $.Values.global.allInOneLocalStorage }}hostPath{{ else }}{{$.Values.storageConfig.type}}{{end}}" #Please ignore this 46 | type: persistentVolumeClaim 47 | hostPath: /opt/deepflow-clickhouse 48 | persistence: 49 | - name: clickhouse-path 50 | accessModes: 51 | - ReadWriteOnce 52 | size: 100Gi 53 | annotations: 54 | storageClass: "" 55 | # selector: 56 | # matchLabels: 57 | # app.kubernetes.io/name: clickhouse 58 | - name: clickhouse-storage-path 59 | accessModes: 60 | - ReadWriteOnce 61 | size: 200Gi 62 | annotations: 63 | storageClass: "" 64 | # selector: 65 | # matchLabels: 66 | # app.kubernetes.io/name: clickhouse 67 | # - name: juicefs 68 | # accessModes: 69 | # - ReadWriteOnce 70 | # size: 200Gi 71 | # annotations: 72 | # storageClass: "juicefs" 73 | # # selector: 74 | # # matchLabels: 75 | # # app.kubernetes.io/name: clickhouse 76 | s3StorageEnabled: false 77 | extraClickhousePath: [] 78 | # - volumeName: juicefs 79 | # mountPath: /juicefs 80 | # diskName: juicefs 81 | # preferNotToMerge: true 82 | 83 | clickhouse: 84 | interserverHttpPort: 9009 85 | maxConcurrentQueries: 2000 86 | ## 单次查询最大内存 (bytes) 87 | maxMemoryUsage: 10000000000 88 | maxQuerySize: 10737418240 89 | maxAstElements: 2000000 90 | maxExpandedAstElements: 2000000 91 | connectTimeout: 500 92 | backgroudPoolSize: 32 93 | service: 94 | ## Configuration for Clickhouse service 95 | ## 96 | annotations: {} 97 | labels: {} 98 | clusterIP: "" 99 | 100 | ## Port for Clickhouse Service to listen on 101 | ## 102 | 103 | ports: 104 | - name: http-port 105 | port: 8123 106 | targetPort: 8123 107 | nodePort: 108 | protocol: TCP 109 | - name: tcp-port 110 | port: 9000 111 | targetPort: 9000 112 | nodePort: 113 | protocol: TCP 114 | - name: interserver-http-port 115 | port: 9009 116 | targetPort: 9009 117 | nodePort: 118 | protocol: TCP 119 | 120 | ## Additional ports to open for server service 121 | additionalPorts: [] 122 | 123 | externalIPs: [] 124 | loadBalancerIP: "" 125 | loadBalancerSourceRanges: [] 126 | 127 | ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints 128 | ## must be Local 129 | externalTrafficPolicy: Cluster 130 | 131 | ## Service type 132 | ## 133 | type: ClusterIP 134 | 135 | 136 | resources: {} 137 | # We usually recommend not to specify default resources and to leave this as a conscious 138 | # choice for the user. This also increases chances charts run on environments with little 139 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 140 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 141 | # limits: 142 | # cpu: 100m 143 | # memory: 128Mi 144 | # requests: 145 | # cpu: 100m 146 | # memory: 128Mi 147 | 148 | nodeSelector: {} 149 | 150 | tolerations: [] 151 | 152 | podAntiAffinityLabelSelector: 153 | - labelSelector: 154 | - key: app 155 | operator: In 156 | values: deepflow 157 | - key: component 158 | operator: In 159 | values: clickhouse 160 | topologyKey: "kubernetes.io/hostname" 161 | podAntiAffinityTermLabelSelector: [] 162 | podAffinityLabelSelector: [] 163 | podAffinityTermLabelSelector: [] 164 | nodeAffinityLabelSelector: [] 165 | nodeAffinityTermLabelSelector: [] -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: mysql 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.000 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "8.0.26" 25 | -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "mysql.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "mysql.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "mysql.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "mysql.labels" -}} 37 | helm.sh/chart: {{ include "mysql.chart" . }} 38 | {{ include "mysql.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "mysql.selectorLabels" -}} 49 | app: deepflow 50 | component: mysql 51 | app.kubernetes.io/name: {{ include "mysql.name" . }} 52 | app.kubernetes.io/instance: {{ .Release.Name }} 53 | {{- end }} 54 | 55 | {{/* 56 | Create the name of the service account to use 57 | */}} 58 | {{- define "mysql.serviceAccountName" -}} 59 | {{- if .Values.serviceAccount.create }} 60 | {{- default (include "mysql.fullname" .) .Values.serviceAccount.name }} 61 | {{- else }} 62 | {{- default "default" .Values.serviceAccount.name }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: {{ include "mysql.fullname" . }} 5 | labels: 6 | {{- include "mysql.labels" . | nindent 4 }} 7 | data: 8 | my.cnf: |- 9 | [client] 10 | default-character-set=utf8 11 | 12 | [mysqld] 13 | default-authentication-plugin=mysql_native_password 14 | 15 | # Network related 16 | bind-address=* 17 | port=30130 18 | 19 | # Enable query cache 20 | innodb_buffer_pool_size=20M 21 | max_connections=1000 22 | wait_timeout=60 23 | 24 | # Replication related 25 | slave_skip_errors=all 26 | 27 | ## Binlog expiration time, default 1 days 28 | binlog_expire_logs_seconds = 86400 29 | init.sql: |- 30 | ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '{{ tpl $.Values.password . }}'; 31 | CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED WITH mysql_native_password BY '{{ tpl $.Values.password . }}'; 32 | GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION; 33 | CREATE USER IF NOT EXISTS 'grafana'@'%' IDENTIFIED WITH mysql_native_password BY '{{ tpl $.Values.password . }}'; 34 | GRANT ALL ON *.* TO 'grafana'@'%' WITH GRANT OPTION; 35 | -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.externalMySQL.enabled }} 2 | {{- if and (not $.Values.global.allInOneLocalStorage ) (eq ( tpl $.Values.storageConfig.generateType . ) "hostPath") (not $.Values.nodeAffinityLabelSelector) }} 3 | {{- fail "You must set nodeAffinityLabelSelector" -}} 4 | {{- end}} 5 | apiVersion: apps/v1 6 | kind: Deployment 7 | metadata: 8 | name: {{ include "mysql.fullname" . }} 9 | labels: 10 | {{- include "mysql.labels" . | nindent 4 }} 11 | spec: 12 | replicas: {{ tpl (toString .Values.replicas) . }} 13 | strategy: 14 | type: Recreate 15 | selector: 16 | matchLabels: 17 | {{- include "mysql.selectorLabels" . | nindent 6 }} 18 | template: 19 | metadata: 20 | annotations: 21 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 22 | {{- with .Values.podAnnotations }} 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | labels: 26 | {{- include "mysql.selectorLabels" . | nindent 8 }} 27 | {{- with .Values.podLabels }} 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | spec: 31 | hostNetwork: {{ tpl (toString .Values.hostNetwork) . }} 32 | dnsPolicy: {{ tpl .Values.dnsPolicy . }} 33 | imagePullSecrets: 34 | {{- with .Values.global.imagePullSecrets }} 35 | {{- toYaml . | nindent 8 }} 36 | {{- end }} 37 | {{- with .Values.imagePullSecrets }} 38 | {{- toYaml . | nindent 8 }} 39 | {{- end }} 40 | securityContext: 41 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 42 | {{- if and (eq ( tpl $.Values.storageConfig.generateType . ) "hostPath") ($.Values.storageConfig.hostPathChownContainerEnabled) }} 43 | {{- if .Values.chmodContainer.enabled }} 44 | initContainers: 45 | - name: hostpath-chown 46 | image: "{{ tpl .Values.image.repository . }}:{{ tpl (toString .Values.image.tag) . }}" 47 | securityContext: 48 | runAsNonRoot: false 49 | runAsUser: 0 50 | command: 51 | - chown 52 | - 27:27 53 | - /var/lib/mysql 54 | volumeMounts: 55 | - mountPath: /var/lib/mysql 56 | name: data-path 57 | {{- end }} 58 | {{- end }} 59 | containers: 60 | - name: mysql 61 | securityContext: 62 | {{- toYaml .Values.securityContext | nindent 12 }} 63 | image: "{{ tpl .Values.image.repository . }}:{{ tpl (toString .Values.image.tag) . }}" 64 | imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} 65 | ports: 66 | - name: tcp 67 | containerPort: 30130 68 | protocol: TCP 69 | livenessProbe: 70 | {{ toYaml .Values.livenessProbe | nindent 12 }} 71 | readinessProbe: 72 | {{ toYaml .Values.readinessProbe | nindent 12 }} 73 | resources: 74 | {{- toYaml .Values.resources | nindent 12 }} 75 | env: 76 | - name: MYSQL_ROOT_PASSWORD 77 | value: {{ tpl .Values.password . }} 78 | - name: MYSQL_DATABASE 79 | value: grafana 80 | - name: TZ 81 | value: "{{ tpl .Values.timezone . }}" 82 | volumeMounts: 83 | - name: mysql-config 84 | mountPath: /etc/my.cnf 85 | subPath: my.cnf 86 | - name: mysql-config-init-sql 87 | mountPath: /docker-entrypoint-initdb.d/init.sql 88 | subPath: init.sql 89 | - mountPath: /var/lib/mysql 90 | name: data-path 91 | volumes: 92 | - name: mysql-config 93 | configMap: 94 | name: {{ include "mysql.fullname" . }} 95 | items: 96 | - key: my.cnf 97 | path: my.cnf 98 | - name: mysql-config-init-sql 99 | configMap: 100 | name: {{ include "mysql.fullname" . }} 101 | items: 102 | - key: init.sql 103 | path: init.sql 104 | - name: data-path 105 | {{- if eq ( tpl .Values.storageConfig.generateType . ) "persistentVolumeClaim" }} 106 | persistentVolumeClaim: 107 | claimName: {{ .Values.storageConfig.persistence.existingClaim | default (printf "%s-data-pvc" (include "mysql.fullname" . )) }} 108 | {{- end }} 109 | {{- if eq ( tpl .Values.storageConfig.generateType . ) "hostPath" }} 110 | hostPath: 111 | type: DirectoryOrCreate 112 | path: {{ tpl .Values.storageConfig.hostPath . }} 113 | {{- end }} 114 | {{- with .Values.nodeSelector }} 115 | nodeSelector: 116 | {{- toYaml . | nindent 8 }} 117 | {{- end }} 118 | affinity: 119 | {{- include "nodeaffinity" . | indent 6 }} 120 | {{- include "podAffinity" . | indent 6 }} 121 | {{- include "podAntiAffinity" . | indent 6 }} 122 | {{- if or .Values.global.tolerations .Values.tolerations }} 123 | tolerations: 124 | {{- if .Values.global.tolerations }} 125 | {{- toYaml .Values.global.tolerations | nindent 8 }} 126 | {{- end }} 127 | {{- if .Values.tolerations }} 128 | {{- toYaml .Values.tolerations | nindent 8 }} 129 | {{- end }} 130 | {{- end }} 131 | {{- end }} 132 | -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/templates/pvc.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (eq ( tpl $.Values.storageConfig.generateType . ) "persistentVolumeClaim") (not .Values.storageConfig.persistence.existingClaim) }} 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | name: {{ include "mysql.fullname" . }}-data-pvc 6 | labels: 7 | {{- include "mysql.labels" . | nindent 4 }} 8 | annotations: 9 | {{- toYaml .Values.storageConfig.persistence.annotations | nindent 4 }} 10 | spec: 11 | {{- if (tpl .Values.storageConfig.persistence.storageClass .) }} 12 | storageClassName: {{ tpl .Values.storageConfig.persistence.storageClass . | quote }} 13 | {{- end }} 14 | accessModes: 15 | - {{ .Values.storageConfig.persistence.accessMode | quote }} 16 | resources: 17 | requests: 18 | storage: {{ .Values.storageConfig.persistence.size | quote }} 19 | {{- end -}} -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/templates/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: {{ include "mysql.fullname" . }} 7 | labels: 8 | {{- include "mysql.labels" . | nindent 4 }} 9 | {{- if .Values.service.annotations }} 10 | annotations: 11 | {{ toYaml .Values.service.annotations | indent 4 }} 12 | {{- end }} 13 | spec: 14 | {{- if .Values.service.clusterIP }} 15 | clusterIP: {{ .Values.service.clusterIP }} 16 | {{- end }} 17 | {{- if .Values.service.externalIPs }} 18 | externalIPs: 19 | {{ toYaml .Values.service.externalIPs | indent 4 }} 20 | {{- end }} 21 | {{- if .Values.service.loadBalancerIP }} 22 | loadBalancerIP: {{ .Values.service.loadBalancerIP }} 23 | {{- end }} 24 | {{- if .Values.service.loadBalancerSourceRanges }} 25 | loadBalancerSourceRanges: 26 | {{- range $cidr := .Values.service.loadBalancerSourceRanges }} 27 | - {{ $cidr }} 28 | {{- end }} 29 | {{- end }} 30 | {{- if ne .Values.service.type "ClusterIP" }} 31 | externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} 32 | {{- end }} 33 | ports: 34 | {{- $serviceType := .Values.service.type -}} 35 | {{- range .Values.service.ports }} 36 | - name: {{ tpl .name $ }} 37 | port: {{ tpl (toString .port) $ }} 38 | targetPort: {{ tpl (toString .targetPort) $ }} 39 | {{- if and (eq $serviceType "NodePort") (.nodePort) }} 40 | nodePort: {{ tpl (toString .nodePort) $ }} 41 | {{- end }} 42 | protocol: {{ tpl .protocol $ }} 43 | {{- end }} 44 | {{- if .Values.service.additionalPorts }} 45 | {{ toYaml .Values.service.additionalPorts | indent 2 }} 46 | {{- end }} 47 | selector: 48 | {{- include "mysql.selectorLabels" . | nindent 4 }} 49 | type: "{{ .Values.service.type }}" -------------------------------------------------------------------------------- /charts/deepflow/charts/mysql/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for mysql. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | global: 5 | podAntiAffinityLabelSelector: [] 6 | podAntiAffinityTermLabelSelector: [] 7 | podAffinityLabelSelector: [] 8 | podAffinityTermLabelSelector: [] 9 | nodeAffinityLabelSelector: [] 10 | nodeAffinityTermLabelSelector: [] 11 | 12 | replicas: 1 13 | 14 | image: 15 | repository: mysql 16 | pullPolicy: IfNotPresent 17 | # Overrides the image tag whose default is the chart appVersion. 18 | tag: 8.0.31 19 | 20 | timezone: "Asia/Shanghai" 21 | hostNetwork: "false" 22 | dnsPolicy: ClusterFirst 23 | imagePullSecrets: [] 24 | nameOverride: "" 25 | fullnameOverride: "" 26 | password: deepflow 27 | 28 | podAnnotations: {} 29 | 30 | podSecurityContext: {} 31 | # fsGroup: 2000 32 | readinessProbe: 33 | tcpSocket: 34 | port: tcp 35 | failureThreshold: 6 36 | initialDelaySeconds: 15 37 | periodSeconds: 10 38 | successThreshold: 1 39 | livenessProbe: 40 | failureThreshold: 6 41 | initialDelaySeconds: 15 42 | periodSeconds: 20 43 | successThreshold: 1 44 | tcpSocket: 45 | port: tcp 46 | timeoutSeconds: 1 47 | 48 | chmodContainer: 49 | enabled: true 50 | securityContext: {} 51 | storageConfig: 52 | generateType: "{{ if $.Values.global.allInOneLocalStorage }}hostPath{{ else }}{{$.Values.storageConfig.type}}{{end}}" #Please ignore this 53 | ## persistentVolumeClaim/hostPath 54 | type: persistentVolumeClaim 55 | hostPath: /opt/deepflow-mysql 56 | persistence: 57 | storageClass: "" 58 | annotations: 59 | "helm.sh/resource-policy": keep 60 | # existingClaim: your-claim-pvc-name 61 | accessMode: ReadWriteOnce 62 | size: 50Gi 63 | 64 | externalMySQL: 65 | enabled: false 66 | hostIP: 192.168.1.1 67 | port: 30130 68 | 69 | service: 70 | ## Configuration for Clickhouse service 71 | ## 72 | annotations: {} 73 | labels: {} 74 | clusterIP: "" 75 | 76 | ## Port for Clickhouse Service to listen on 77 | ## 78 | 79 | ports: 80 | - name: tcp 81 | port: 30130 82 | targetPort: 30130 83 | nodePort: 84 | protocol: TCP 85 | ## Additional ports to open for server service 86 | additionalPorts: [] 87 | 88 | externalIPs: [] 89 | loadBalancerIP: "" 90 | loadBalancerSourceRanges: [] 91 | 92 | ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints 93 | ## must be Local 94 | externalTrafficPolicy: Cluster 95 | 96 | ## Service type 97 | ## 98 | type: ClusterIP 99 | 100 | 101 | resources: {} 102 | # We usually recommend not to specify default resources and to leave this as a conscious 103 | # choice for the user. This also increases chances charts run on environments with little 104 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 105 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 106 | # limits: 107 | # cpu: 100m 108 | # memory: 128Mi 109 | # requests: 110 | # cpu: 100m 111 | # memory: 128Mi 112 | 113 | nodeSelector: {} 114 | 115 | tolerations: [] 116 | 117 | podAntiAffinityLabelSelector: [] 118 | podAntiAffinityTermLabelSelector: [] 119 | podAffinityLabelSelector: [] 120 | podAffinityTermLabelSelector: [] 121 | nodeAffinityLabelSelector: [] 122 | nodeAffinityTermLabelSelector: [] 123 | 124 | -------------------------------------------------------------------------------- /charts/deepflow/charts/stella-agent-ce/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | description: An automated observability platform for cloud-native developers. 3 | name: stella-agent-ce 4 | engine: gotpl 5 | type: application 6 | version: 6.5.001 7 | appVersion: "6.5.0" 8 | kubeVersion: ">=1.16.0-0" 9 | sources: 10 | - https://github.com/deepflowio/deepflow 11 | home: https://github.com/deepflowio/deepflow 12 | keywords: 13 | - deepflow 14 | icon: https://raw.githubusercontent.com/deepflowio/deepflow-charts/main/deepflow.svg 15 | maintainers: 16 | - name: deepflow 17 | url: https://github.com/deepflowio/deepflow 18 | -------------------------------------------------------------------------------- /charts/deepflow/charts/stella-agent-ce/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "stella-agent-ce.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "stella-agent-ce.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "stella-agent-ce.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "stella-agent-ce.labels" -}} 37 | helm.sh/chart: {{ include "stella-agent-ce.chart" . }} 38 | {{ include "stella-agent-ce.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "stella-agent-ce.selectorLabels" -}} 49 | app: deepflow 50 | component: stella-agent-ce 51 | app.kubernetes.io/name: {{ include "stella-agent-ce.name" . }} 52 | app.kubernetes.io/instance: {{ .Release.Name }} 53 | {{- end }} 54 | 55 | {{/* 56 | Create the name of the service account to use 57 | */}} 58 | {{- define "stella-agent-ce.serviceAccountName" -}} 59 | {{- if .Values.serviceAccount.create }} 60 | {{- default (include "stella-agent-ce.fullname" .) .Values.serviceAccount.name }} 61 | {{- else }} 62 | {{- default "default" .Values.serviceAccount.name }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/deepflow/charts/stella-agent-ce/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "stella-agent-ce.fullname" . }} 5 | labels: 6 | {{- include "stella-agent-ce.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ tpl (toString .Values.replicas) . }} 9 | selector: 10 | matchLabels: 11 | {{- include "stella-agent-ce.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | annotations: 15 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 16 | {{- with .Values.podAnnotations }} 17 | {{- toYaml . | nindent 8 }} 18 | {{- end }} 19 | labels: 20 | {{- include "stella-agent-ce.selectorLabels" . | nindent 8 }} 21 | {{- with .Values.podLabels }} 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | spec: 25 | hostNetwork: {{ tpl (toString .Values.hostNetwork) . }} 26 | dnsPolicy: {{ tpl .Values.dnsPolicy . }} 27 | imagePullSecrets: 28 | {{- with .Values.global.imagePullSecrets }} 29 | {{- toYaml . | nindent 8 }} 30 | {{- end }} 31 | {{- with .Values.imagePullSecrets }} 32 | {{- toYaml . | nindent 8 }} 33 | {{- end }} 34 | securityContext: 35 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 36 | containers: 37 | - name: stella-agent-ce 38 | securityContext: 39 | {{- toYaml .Values.securityContext | nindent 12 }} 40 | image: "{{ tpl .Values.image.repository . }}:{{ tpl (toString .Values.image.tag) . }}" 41 | imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} 42 | ports: 43 | - name: http 44 | containerPort: 20831 45 | protocol: TCP 46 | livenessProbe: 47 | {{ toYaml .Values.livenessProbe | nindent 12 }} 48 | readinessProbe: 49 | {{ toYaml .Values.readinessProbe | nindent 12 }} 50 | resources: 51 | {{- toYaml .Values.resources | nindent 12 }} 52 | volumeMounts: 53 | - name: config 54 | mountPath: /etc/web/df-llm-agent.yaml 55 | subPath: df-llm-agent.yaml 56 | volumes: 57 | - name: config 58 | configMap: 59 | name: {{ include "stella-agent-ce.fullname" . }} 60 | items: 61 | - key: df-llm-agent.yaml 62 | path: df-llm-agent.yaml 63 | {{- with .Values.nodeSelector }} 64 | nodeSelector: 65 | {{- toYaml . | nindent 8 }} 66 | {{- end }} 67 | affinity: 68 | {{- include "nodeaffinity" . | indent 6 }} 69 | {{- include "podAffinity" . | indent 6 }} 70 | {{- include "podAntiAffinity" . | indent 6 }} 71 | {{- if or .Values.global.tolerations .Values.tolerations }} 72 | tolerations: 73 | {{- if .Values.global.tolerations }} 74 | {{- toYaml .Values.global.tolerations | nindent 8 }} 75 | {{- end }} 76 | {{- if .Values.tolerations }} 77 | {{- toYaml .Values.tolerations | nindent 8 }} 78 | {{- end }} 79 | {{- end }} 80 | -------------------------------------------------------------------------------- /charts/deepflow/charts/stella-agent-ce/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "stella-agent-ce.fullname" . }} 5 | labels: 6 | {{- include "stella-agent-ce.labels" . | nindent 4 }} 7 | {{- if .Values.service.annotations }} 8 | annotations: 9 | {{ toYaml .Values.service.annotations | indent 4 }} 10 | {{- end }} 11 | spec: 12 | {{- if .Values.service.clusterIP }} 13 | clusterIP: {{ .Values.service.clusterIP }} 14 | {{- end }} 15 | {{- if .Values.service.externalIPs }} 16 | externalIPs: 17 | {{ toYaml .Values.service.externalIPs | indent 4 }} 18 | {{- end }} 19 | {{- if .Values.service.loadBalancerIP }} 20 | loadBalancerIP: {{ .Values.service.loadBalancerIP }} 21 | {{- end }} 22 | {{- if .Values.service.loadBalancerSourceRanges }} 23 | loadBalancerSourceRanges: 24 | {{- range $cidr := .Values.service.loadBalancerSourceRanges }} 25 | - {{ $cidr }} 26 | {{- end }} 27 | {{- end }} 28 | {{- if ne .Values.service.type "ClusterIP" }} 29 | externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} 30 | {{- end }} 31 | ports: 32 | {{- $serviceType := .Values.service.type -}} 33 | {{- range .Values.service.ports }} 34 | - name: {{ tpl .name $ }} 35 | port: {{ tpl (toString .port) $ }} 36 | targetPort: {{ tpl (toString .targetPort) $ }} 37 | {{- if and (eq $serviceType "NodePort") (.nodePort) }} 38 | nodePort: {{ tpl (toString .nodePort) $ }} 39 | {{- end }} 40 | protocol: {{ tpl .protocol $ }} 41 | {{- end }} 42 | {{- if .Values.service.additionalPorts }} 43 | {{ toYaml .Values.service.additionalPorts | indent 2 }} 44 | {{- end }} 45 | selector: 46 | {{- include "stella-agent-ce.selectorLabels" . | nindent 4 }} 47 | type: "{{ .Values.service.type }}" -------------------------------------------------------------------------------- /charts/deepflow/charts/stella-agent-ce/values.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | podAntiAffinityLabelSelector: [] 3 | podAntiAffinityTermLabelSelector: [] 4 | podAffinityLabelSelector: [] 5 | podAffinityTermLabelSelector: [] 6 | nodeAffinityLabelSelector: [] 7 | nodeAffinityTermLabelSelector: [] 8 | 9 | replicas: 1 10 | 11 | hostNetwork: "false" 12 | dnsPolicy: ClusterFirst 13 | imagePullSecrets: [] 14 | nameOverride: "" 15 | fullnameOverride: "" 16 | podAnnotations: {} 17 | 18 | image: 19 | repository: deepflowce/deepflowio-stella-agent-ce 20 | pullPolicy: Always 21 | # Overrides the image tag whose default is the chart appVersion. 22 | tag: latest 23 | 24 | podSecurityContext: {} 25 | # fsGroup: 2000 26 | 27 | securityContext: 28 | # privileged: true 29 | # capabilities: 30 | # drop: 31 | # - ALL 32 | # readOnlyRootFilesystem: false 33 | # runAsNonRoot: false 34 | # runAsUser: 0 35 | 36 | service: 37 | ## Configuration for ClickHouse service 38 | ## 39 | annotations: {} 40 | labels: {} 41 | clusterIP: "" 42 | 43 | ## Port for ClickHouse Service to listen on 44 | ## 45 | 46 | ports: 47 | - name: tcp 48 | port: 20831 49 | targetPort: 20831 50 | nodePort: 51 | protocol: TCP 52 | ## Additional ports to open for server service 53 | additionalPorts: [] 54 | 55 | externalIPs: [] 56 | loadBalancerIP: "" 57 | loadBalancerSourceRanges: [] 58 | 59 | ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints 60 | 61 | externalTrafficPolicy: Cluster 62 | 63 | ## Service type 64 | ## 65 | type: ClusterIP 66 | 67 | readinessProbe: 68 | httpGet: 69 | path: /v1/health/ 70 | port: http 71 | failureThreshold: 10 72 | initialDelaySeconds: 15 73 | periodSeconds: 10 74 | successThreshold: 1 75 | livenessProbe: 76 | failureThreshold: 6 77 | initialDelaySeconds: 15 78 | periodSeconds: 20 79 | successThreshold: 1 80 | httpGet: 81 | path: /v1/health/ 82 | port: http 83 | timeoutSeconds: 1 84 | 85 | configmap: 86 | df-llm-agent.yaml: 87 | daemon: true 88 | api_timeout: 500 89 | sql_show: false 90 | log_file: "/var/log/df-llm-agent.log" 91 | log_level: "info" 92 | instance_path: "/root/df-llm-agent" 93 | mysql: 94 | host: "{{ if $.Values.global.externalMySQL.enabled }}{{$.Values.global.externalMySQL.ip}}{{ else }}{{ $.Release.Name }}-mysql{{end}}" 95 | port: "{{ if $.Values.global.externalMySQL.enabled }}{{$.Values.global.externalMySQL.port}}{{ else }}30130{{end}}" 96 | user_password: "{{ if $.Values.global.externalMySQL.enabled }}{{$.Values.global.externalMySQL.password}}{{ else }}{{ .Values.global.password.mysql }}{{end}}" 97 | database: "deepflow_llm" 98 | 99 | resources: {} 100 | # We usually recommend not to specify default resources and to leave this as a conscious 101 | # choice for the user. This also increases chances charts run on environments with little 102 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 103 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 104 | # limits: 105 | # cpu: 100m 106 | # memory: 128Mi 107 | # requests: 108 | # cpu: 100m 109 | # memory: 128Mi 110 | 111 | nodeSelector: {} 112 | 113 | tolerations: [] 114 | 115 | podAntiAffinityLabelSelector: [] 116 | podAntiAffinityTermLabelSelector: [] 117 | podAffinityLabelSelector: [] 118 | podAffinityTermLabelSelector: [] 119 | nodeAffinityLabelSelector: [] 120 | nodeAffinityTermLabelSelector: [] -------------------------------------------------------------------------------- /charts/deepflow/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | ██████╗ ███████╗███████╗██████╗ ███████╗██╗ ██████╗ ██╗ ██╗ 2 | ██╔══██╗██╔════╝██╔════╝██╔══██╗██╔════╝██║ ██╔═══██╗██║ ██║ 3 | ██║ ██║█████╗ █████╗ ██████╔╝█████╗ ██║ ██║ ██║██║ █╗ ██║ 4 | ██║ ██║██╔══╝ ██╔══╝ ██╔═══╝ ██╔══╝ ██║ ██║ ██║██║███╗██║ 5 | ██████╔╝███████╗███████╗██║ ██║ ███████╗╚██████╔╝╚███╔███╔╝ 6 | ╚═════╝ ╚══════╝╚══════╝╚═╝ ╚═╝ ╚══════╝ ╚═════╝ ╚══╝╚══╝ 7 | 8 | An automated observability platform for cloud-native developers. 9 | 10 | # deepflow-agent Port for receiving trace, metrics, and log 11 | 12 | deepflow-agent service: deepflow-agent.{{ $.Release.Namespace }} 13 | deepflow-agent Host listening port: {{ tpl (toString (index .Values "deepflow-agent" "externalAgentHttpProxyPort")) . }} 14 | 15 | # Get the Grafana URL to visit by running these commands in the same shell 16 | {{ if contains "NodePort" .Values.grafana.service.type }} 17 | NODE_PORT=$(kubectl get --namespace {{ $.Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ $.Release.Name }}-grafana) 18 | NODE_IP=$(kubectl get nodes -o jsonpath="{.items[0].status.addresses[0].address}") 19 | echo -e "Grafana URL: http://$NODE_IP:$NODE_PORT \nGrafana auth: admin:{{ tpl .Values.grafana.adminPassword . }}" 20 | {{ else if contains "LoadBalancer" .Values.grafana.service.type -}} 21 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 22 | You can watch the status of by running 'kubectl get svc --namespace {{ $.Release.Namespace }} -w {{ $.Release.Name }}-grafana' 23 | SERVICE_IP=$(kubectl get svc --namespace {{ $.Release.Namespace }} {{ $.Release.Name }}-grafana -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 24 | http://$SERVICE_IP:{{ .Values.grafana.service.port -}} 25 | {{ else if contains "ClusterIP" .Values.grafana.service.type }} 26 | POD_NAME=$(kubectl get pods --namespace {{ $.Release.Namespace }} -l "app.kubernetes.io/name={{ template "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 27 | kubectl --namespace {{ $.Release.Namespace }} port-forward $POD_NAME 3000 28 | {{- end }} -------------------------------------------------------------------------------- /charts/deepflow/templates/RBAC.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "deepflow.fullname" . }}-server 5 | --- 6 | kind: Role 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | metadata: 9 | name: {{ include "deepflow.fullname" . }}-server 10 | namespace: {{ .Release.Namespace }} 11 | rules: 12 | - apiGroups: 13 | - "" 14 | resources: 15 | - endpoints 16 | - services 17 | verbs: 18 | - list 19 | - get 20 | - watch 21 | - create 22 | - update 23 | - apiGroups: 24 | - "" 25 | resources: 26 | - pods 27 | verbs: 28 | - list 29 | - get 30 | - watch 31 | - apiGroups: 32 | - coordination.k8s.io 33 | resources: 34 | - leases 35 | verbs: 36 | - '*' 37 | --- 38 | kind: RoleBinding 39 | apiVersion: rbac.authorization.k8s.io/v1 40 | metadata: 41 | name: {{ include "deepflow.fullname" . }}-server 42 | subjects: 43 | - kind: ServiceAccount 44 | name: {{ include "deepflow.fullname" . }}-server 45 | namespace: {{ .Release.Namespace }} 46 | apiGroup: "" 47 | roleRef: 48 | kind: Role 49 | name: {{ include "deepflow.fullname" . }}-server 50 | apiGroup: "" -------------------------------------------------------------------------------- /charts/deepflow/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "deepflow.name" -}} 5 | {{- $deepflowChartName := "deepflow" }} 6 | {{- default $deepflowChartName .Values.nameOverride | trunc 63 | trimSuffix "-" }} 7 | {{- end }} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "deepflow.fullname" -}} 15 | {{- if .Values.fullnameOverride }} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 17 | {{- else }} 18 | {{- $deepflowChartName := "deepflow" }} 19 | {{- $name := default $deepflowChartName .Values.nameOverride }} 20 | {{- if contains $name .Release.Name }} 21 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 22 | {{- else }} 23 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | 28 | {{/* 29 | Create chart name and version as used by the chart label. 30 | */}} 31 | {{- define "deepflow.chart" -}} 32 | {{- $deepflowChartName := "deepflow" }} 33 | {{- printf "%s-%s" $deepflowChartName .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 34 | {{- end }} 35 | 36 | {{/* 37 | Common labels 38 | */}} 39 | {{- define "deepflow.labels" -}} 40 | helm.sh/chart: {{ include "deepflow.chart" . }} 41 | {{ include "deepflow.selectorLabels" . }} 42 | {{- if .Chart.AppVersion }} 43 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 44 | {{- end }} 45 | app.kubernetes.io/managed-by: {{ .Release.Service }} 46 | {{- end }} 47 | 48 | {{/* 49 | Selector labels 50 | */}} 51 | {{- define "deepflow.selectorLabels" -}} 52 | app: deepflow 53 | app.kubernetes.io/name: {{ include "deepflow.name" . }} 54 | app.kubernetes.io/instance: {{ .Release.Name }} 55 | {{- end }} 56 | 57 | {{/* 58 | Common labels 59 | */}} 60 | {{- define "deepflow-server.labels" -}} 61 | helm.sh/chart: {{ include "deepflow.chart" . }} 62 | {{ include "deepflow.selectorLabels" . }} 63 | {{- if .Chart.AppVersion }} 64 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 65 | {{- end }} 66 | app.kubernetes.io/managed-by: {{ .Release.Service }} 67 | {{- end }} 68 | 69 | {{/* 70 | Selector labels 71 | */}} 72 | {{- define "deepflow-server.selectorLabels" -}} 73 | app: deepflow 74 | component: deepflow-server 75 | app.kubernetes.io/name: {{ include "deepflow.name" . }} 76 | app.kubernetes.io/instance: {{ .Release.Name }} 77 | {{- end }} 78 | 79 | {{/* 80 | Common labels 81 | */}} 82 | {{- define "deepflow-app.labels" -}} 83 | helm.sh/chart: {{ include "deepflow.chart" . }} 84 | {{ include "deepflow-app.selectorLabels" . }} 85 | {{- if .Chart.AppVersion }} 86 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 87 | {{- end }} 88 | app.kubernetes.io/managed-by: {{ .Release.Service }} 89 | {{- end }} 90 | 91 | {{/* 92 | Selector labels 93 | */}} 94 | {{- define "deepflow-app.selectorLabels" -}} 95 | app: deepflow 96 | component: deepflow-app 97 | app.kubernetes.io/name: {{ include "deepflow.name" . }} 98 | app.kubernetes.io/instance: {{ .Release.Name }} 99 | {{- end }} 100 | -------------------------------------------------------------------------------- /charts/deepflow/templates/app-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "deepflow.fullname" . }}-app 5 | labels: 6 | {{- include "deepflow-app.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ tpl (toString .Values.app.replicas) . }} 9 | selector: 10 | matchLabels: 11 | {{- include "deepflow-app.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | annotations: 15 | {{- with .Values.app.podAnnotations }} 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 19 | checksum/customConfig: {{ sha256sum (print (tpl (toYaml .Values.configmap) $)) }} 20 | labels: 21 | {{- include "deepflow-app.selectorLabels" . | nindent 8 }} 22 | {{- with .Values.app.podLabels }} 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | spec: 26 | hostNetwork: {{ tpl .Values.app.hostNetwork . }} 27 | dnsPolicy: {{ tpl .Values.app.dnsPolicy . }} 28 | imagePullSecrets: 29 | {{- with .Values.global.imagePullSecrets }} 30 | {{- toYaml . | nindent 8 }} 31 | {{- end }} 32 | {{- with .Values.imagePullSecrets }} 33 | {{- toYaml . | nindent 8 }} 34 | {{- end }} 35 | securityContext: 36 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 37 | containers: 38 | - name: deepflow-app 39 | securityContext: 40 | {{- toYaml .Values.securityContext | nindent 12 }} 41 | image: "{{ tpl .Values.image.app.repository . }}:{{ tpl .Values.image.app.tag . }}" 42 | imagePullPolicy: "{{ tpl .Values.image.app.pullPolicy . }}" 43 | ports: 44 | - name: app 45 | containerPort: 20418 46 | protocol: TCP 47 | livenessProbe: 48 | {{ toYaml .Values.app.livenessProbe | nindent 12 }} 49 | readinessProbe: 50 | {{ toYaml .Values.app.readinessProbe | nindent 12 }} 51 | resources: 52 | {{- toYaml .Values.app.resources | nindent 12 }} 53 | volumeMounts: 54 | - name: app-config 55 | mountPath: /etc/deepflow/app.yaml 56 | subPath: app.yaml 57 | env: 58 | - name: K8S_NODE_IP_FOR_DEEPFLOW 59 | valueFrom: 60 | fieldRef: 61 | fieldPath: status.hostIP 62 | - name: K8S_NODE_NAME_FOR_DEEPFLOW 63 | valueFrom: 64 | fieldRef: 65 | fieldPath: spec.nodeName 66 | - name: K8S_POD_NAME_FOR_DEEPFLOW 67 | valueFrom: 68 | fieldRef: 69 | fieldPath: metadata.name 70 | - name: K8S_POD_IP_FOR_DEEPFLOW 71 | valueFrom: 72 | fieldRef: 73 | fieldPath: status.podIP 74 | - name: TZ 75 | value: "{{ tpl .Values.timezone . }}" 76 | volumes: 77 | - name: app-config 78 | configMap: 79 | name: {{ include "deepflow.fullname" . }} 80 | items: 81 | - key: app.yaml 82 | path: app.yaml 83 | {{- with .Values.app.nodeSelector }} 84 | nodeSelector: 85 | {{- toYaml . | nindent 8 }} 86 | {{- end }} 87 | affinity: 88 | {{- include "appNodeaffinity" . | indent 6 }} 89 | {{- include "appPodAffinity" . | indent 6 }} 90 | {{- include "appPodAntiAffinity" . | indent 6 }} 91 | {{- if or .Values.global.tolerations .Values.tolerations }} 92 | tolerations: 93 | {{- if .Values.global.tolerations }} 94 | {{- toYaml .Values.global.tolerations | nindent 8 }} 95 | {{- end }} 96 | {{- if .Values.tolerations }} 97 | {{- toYaml .Values.tolerations | nindent 8 }} 98 | {{- end }} 99 | {{- end }} 100 | -------------------------------------------------------------------------------- /charts/deepflow/templates/external-clickhouse-svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.global.externalClickHouse.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "deepflow.fullname" . }}-external-clickhouse 6 | labels: 7 | {{- include "deepflow.labels" . | nindent 4 }} 8 | spec: 9 | ports: 10 | - name: tcp-port 11 | port: 9000 12 | targetPort: tcp-port 13 | protocol: TCP 14 | type: ClusterIP 15 | --- 16 | apiVersion: v1 17 | kind: Endpoints 18 | metadata: 19 | name: {{ include "deepflow.fullname" . }}-external-clickhouse 20 | subsets: 21 | {{- range .Values.global.externalClickHouse.hosts }} 22 | - addresses: 23 | - ip: {{ tpl (toString .ip) $ }} 24 | ports: 25 | - name: tcp-port 26 | port: {{ tpl (toString .port) $ }} 27 | protocol: TCP 28 | {{- end }} 29 | 30 | {{/* 31 | --- 32 | {{- if eq .Values.global.externalClickhouse.type "cep" }} 33 | apiVersion: sealos.io/v1beta1 34 | kind: ClusterEndpoint 35 | metadata: 36 | name: wordpress 37 | namespace: default 38 | spec: 39 | hosts: 40 | - 172.18.191.215 41 | periodSeconds: 10 42 | ports: 43 | - name: wp-https 44 | protocol: TCP 45 | port: 38081 46 | targetPort: 443 47 | tcpSocket: 48 | enable: true 49 | timeoutSeconds: 1 50 | failureThreshold: 3 51 | successThreshold: 1 52 | - name: wp-http 53 | protocol: TCP 54 | port: 38082 55 | targetPort: 80 56 | httpGet: 57 | path: /healthz 58 | scheme: http 59 | timeoutSeconds: 1 60 | failureThreshold: 3 61 | successThreshold: 1 62 | - name: wp-udp 63 | protocol: UDP 64 | port: 38003 65 | targetPort: 1234 66 | udpSocket: 67 | enable: true 68 | data: "This is flag data for UDP svc test" 69 | timeoutSeconds: 1 70 | failureThreshold: 3 71 | successThreshold: 1 72 | - name: wp-grpc 73 | protocol: TCP 74 | port: 38083 75 | targetPort: 8080 76 | grpc: 77 | enable: true 78 | timeoutSeconds: 1 79 | failureThreshold: 3 80 | successThreshold: 1 81 | {{- end }} 82 | {{- end }} 83 | */}} 84 | {{- end }} -------------------------------------------------------------------------------- /charts/deepflow/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "deepflow.fullname" . }}-server 5 | labels: 6 | {{- include "deepflow-server.labels" . | nindent 4 }} 7 | {{- if .Values.server.service.annotations }} 8 | annotations: 9 | {{ toYaml .Values.server.service.annotations | indent 4 }} 10 | {{- end }} 11 | spec: 12 | {{- if .Values.server.service.clusterIP }} 13 | clusterIP: {{ .Values.server.service.clusterIP }} 14 | {{- end }} 15 | {{- if .Values.server.service.externalIPs }} 16 | externalIPs: 17 | {{ toYaml .Values.server.service.externalIPs | indent 4 }} 18 | {{- end }} 19 | {{- if .Values.server.service.loadBalancerIP }} 20 | loadBalancerIP: {{ .Values.server.service.loadBalancerIP }} 21 | {{- end }} 22 | {{- if .Values.server.service.loadBalancerSourceRanges }} 23 | loadBalancerSourceRanges: 24 | {{- range $cidr := .Values.server.service.loadBalancerSourceRanges }} 25 | - {{ $cidr }} 26 | {{- end }} 27 | {{- end }} 28 | {{- if ne .Values.server.service.type "ClusterIP" }} 29 | externalTrafficPolicy: {{ .Values.server.service.externalTrafficPolicy }} 30 | {{- end }} 31 | ports: 32 | {{- $serviceType := .Values.server.service.type -}} 33 | {{- range .Values.server.service.ports }} 34 | - name: {{ tpl .name $ }} 35 | port: {{ tpl (toString .port) $ }} 36 | targetPort: {{ tpl (toString .targetPort) $ }} 37 | {{- if and (eq $serviceType "NodePort") (.nodePort) }} 38 | nodePort: {{ tpl (toString .nodePort) $ }} 39 | {{- end }} 40 | protocol: {{ tpl .protocol $ }} 41 | {{- end }} 42 | {{- if .Values.server.service.additionalPorts }} 43 | {{ toYaml .Values.server.service.additionalPorts | indent 2 }} 44 | {{- end }} 45 | selector: 46 | {{- include "deepflow-server.selectorLabels" . | nindent 4 }} 47 | type: "{{ .Values.server.service.type }}" 48 | --- 49 | apiVersion: v1 50 | kind: Service 51 | metadata: 52 | name: {{ include "deepflow.fullname" . }}-app 53 | labels: 54 | {{- include "deepflow-app.labels" . | nindent 4 }} 55 | {{- if .Values.app.service.annotations }} 56 | annotations: 57 | {{ toYaml .Values.app.service.annotations | indent 4 }} 58 | {{- end }} 59 | spec: 60 | {{- if .Values.app.service.clusterIP }} 61 | clusterIP: {{ .Values.app.service.clusterIP }} 62 | {{- end }} 63 | {{- if .Values.app.service.externalIPs }} 64 | externalIPs: 65 | {{ toYaml .Values.app.service.externalIPs | indent 4 }} 66 | {{- end }} 67 | {{- if .Values.app.service.loadBalancerIP }} 68 | loadBalancerIP: {{ .Values.app.service.loadBalancerIP }} 69 | {{- end }} 70 | {{- if .Values.app.service.loadBalancerSourceRanges }} 71 | loadBalancerSourceRanges: 72 | {{- range $cidr := .Values.app.service.loadBalancerSourceRanges }} 73 | - {{ $cidr }} 74 | {{- end }} 75 | {{- end }} 76 | {{- if ne .Values.app.service.type "ClusterIP" }} 77 | externalTrafficPolicy: {{ .Values.app.service.externalTrafficPolicy }} 78 | {{- end }} 79 | ports: 80 | {{- $serviceType := .Values.app.service.type -}} 81 | {{- range .Values.app.service.ports }} 82 | - name: {{ tpl .name $ }} 83 | port: {{ tpl (toString .port) $ }} 84 | targetPort: {{ tpl (toString .targetPort) $ }} 85 | {{- if and (eq $serviceType "NodePort") (.nodePort) }} 86 | nodePort: {{ tpl (toString .nodePort) $ }} 87 | {{- end }} 88 | protocol: {{ tpl .protocol $ }} 89 | {{- end }} 90 | {{- if .Values.app.service.additionalPorts }} 91 | {{ toYaml .Values.app.service.additionalPorts | indent 2 }} 92 | {{- end }} 93 | selector: 94 | {{- include "deepflow-app.selectorLabels" . | nindent 4 }} 95 | type: "{{ .Values.app.service.type }}" -------------------------------------------------------------------------------- /commit-template: -------------------------------------------------------------------------------- 1 | (Title) 2 | 3 | **Phenomenon and reproduction steps** 4 | 5 | (Describe the phenomenon and scenario of the bug) 6 | 7 | **Root cause and solution** 8 | 9 | (Describe the cause of the bug and corresponding solution) 10 | 11 | **Impactions** 12 | 13 | (Visible changes after this fix) 14 | 15 | **Test method** 16 | 17 | (Test method to ensure the bug is fixed) 18 | 19 | **Affected branch(es)** 20 | 21 | * main 22 | 23 | **Checklist** 24 | 25 | - [ ] Dependencies update required 26 | - [ ] Common bug (similar problem in other repo) 27 | -------------------------------------------------------------------------------- /deepflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepflowio/deepflow-charts/95c26ff391f36643a9ebb8c35684e6927dacc11f/deepflow.png -------------------------------------------------------------------------------- /test-values.yaml: -------------------------------------------------------------------------------- 1 | mysql: 2 | image: 3 | ## MySQL Dockerhub Image repository: mysql 4 | repository: mysql 5 | clickhouse: 6 | image: 7 | ## Clickhouse Dockerhub Image repository: clickhouse/clickhouse-server 8 | repository: clickhouse/clickhouse-server 9 | grafana: 10 | image: 11 | registry: docker.io 12 | repository: grafana/grafana 13 | assertNoLeakedSecrets: false 14 | # 15 | 16 | byconity: 17 | enabled: true 18 | nameOverride: "" 19 | fullnameOverride: "" 20 | 21 | image: 22 | repository: byconity/byconity 23 | tag: 1.0.0 24 | imagePullPolicy: IfNotPresent 25 | fdbShell: 26 | image: 27 | repository: foundationdb 28 | 29 | # For more detailed usage, please check fdb-kubernetes-operator API doc: https://github.com/FoundationDB/fdb-kubernetes-operator/blob/main/docs/cluster_spec.md 30 | fdb: 31 | enabled: true 32 | enableCliPod: true 33 | version: 7.1.15 34 | clusterSpec: 35 | mainContainer: 36 | imageConfigs: 37 | - version: 7.1.15 38 | baseImage: foundationdb/foundationdb 39 | tag: 7.1.15 40 | sidecarContainer: 41 | imageConfigs: 42 | - version: 7.1.15 43 | baseImage: foundationdb/foundationdb-kubernetes-sidecar 44 | tag: 7.1.15-1 45 | processCounts: 46 | stateless: 3 47 | log: 3 48 | storage: 3 49 | processes: 50 | general: 51 | volumeClaimTemplate: 52 | spec: 53 | storageClassName: openebs-hostpath #replace to your storageClassName 54 | resources: 55 | requests: 56 | storage: 20Gi 57 | 58 | fdb-operator: 59 | enabled: true 60 | resources: 61 | limits: 62 | cpu: 1 63 | memory: 512Mi 64 | requests: 65 | cpu: 1 66 | memory: 512Mi 67 | affinity: 68 | nodeAffinity: 69 | requiredDuringSchedulingIgnoredDuringExecution: 70 | nodeSelectorTerms: 71 | - matchExpressions: 72 | - key: tsdb 73 | operator: In 74 | values: 75 | - enable 76 | image: 77 | repository: foundationdb/fdb-kubernetes-operator 78 | tag: v1.9.0 79 | pullPolicy: IfNotPresent 80 | initContainerImage: 81 | repository: foundationdb/foundationdb-kubernetes-sidecar 82 | initContainers: 83 | 6.2: 84 | image: 85 | repository: foundationdb/foundationdb-kubernetes-sidecar 86 | tag: 6.2.30-1 87 | pullPolicy: IfNotPresent 88 | 6.3: 89 | image: 90 | repository: foundationdb/foundationdb-kubernetes-sidecar 91 | tag: 6.3.23-1 92 | pullPolicy: IfNotPresent 93 | 7.1: 94 | image: 95 | repository: foundationdb/foundationdb-kubernetes-sidecar 96 | tag: 7.1.15-1 97 | pullPolicy: IfNotPresent 98 | hdfs: 99 | enabled: false --------------------------------------------------------------------------------