├── druid ├── templates │ ├── simple-node-deployment.yaml │ ├── config │ │ ├── broker │ │ │ ├── _main.config │ │ │ ├── _jvm.config │ │ │ └── _runtime.properties │ │ ├── overlord │ │ │ ├── _main.config │ │ │ ├── _jvm.config │ │ │ └── _runtime.properties │ │ ├── coordinator │ │ │ ├── _main.config │ │ │ ├── _jvm.config │ │ │ └── _runtime.properties │ │ ├── historical │ │ │ ├── _main.config │ │ │ ├── _jvm.config │ │ │ └── _runtime.properties │ │ ├── middlemanager │ │ │ ├── _main.config │ │ │ ├── _jvm.config │ │ │ └── _runtime.properties │ │ ├── _config.yaml │ │ ├── _metrics-server.json │ │ └── _common-runtime.properties │ ├── pivot_cm.yaml │ ├── tranquility_cm.yaml │ ├── secrets.yml │ ├── volume-claim.yaml │ ├── _helpers.tpl │ ├── NOTES.txt │ ├── pivot.yaml │ ├── tranquility.yaml │ ├── broker.yaml │ ├── common_cm.yaml │ ├── middlemanager.yaml │ ├── overlord.yaml │ ├── coordinator.yaml │ └── historical.yaml ├── charts │ └── utils │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ ├── templates │ │ ├── _include.tpl │ │ └── _helpers.tpl │ │ └── .helmignore ├── Chart.yaml ├── requirements.yaml ├── requirements.lock └── values.yaml ├── dockerfiles ├── metadata-mysql │ ├── Dockerfile │ ├── charset.cnf │ └── Deployment.yml ├── druid-quickstart │ ├── Chart.yaml │ ├── templates │ │ ├── service.yaml │ │ ├── _helpers.tpl │ │ ├── deployment.yaml │ │ └── NOTES.txt │ ├── .helmignore │ ├── manifests │ │ └── druid-historical-rc.yaml │ └── values.yaml ├── overlord │ ├── Dockerfile │ └── Deployment.yml ├── historical │ ├── Deployment.yml │ ├── Dockerfile │ └── Single_Pod.yml ├── tranquility │ └── Dockerfile ├── broker │ ├── Dockerfile │ └── Deployment.yml ├── pivot │ ├── Dockerfile │ ├── config.yaml │ └── Deployment.yml ├── local-zk │ ├── Dockerfile │ └── Deployment.yml ├── coordinator │ ├── Dockerfile │ └── Deployment.yml ├── middleManager │ ├── Dockerfile │ └── Deployment.yml ├── build_images.sh ├── base │ ├── Dockerfile │ ├── run-druid │ └── common.runtime.properties └── quickstart │ ├── Deployment.yml │ └── Dockerfile ├── .gitignore └── README.md /druid/templates/simple-node-deployment.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /druid/templates/config/broker/_main.config: -------------------------------------------------------------------------------- 1 | io.druid.cli.Main server broker 2 | -------------------------------------------------------------------------------- /druid/templates/config/overlord/_main.config: -------------------------------------------------------------------------------- 1 | io.druid.cli.Main server overlord 2 | -------------------------------------------------------------------------------- /druid/templates/config/coordinator/_main.config: -------------------------------------------------------------------------------- 1 | io.druid.cli.Main server coordinator 2 | -------------------------------------------------------------------------------- /druid/templates/config/historical/_main.config: -------------------------------------------------------------------------------- 1 | io.druid.cli.Main server historical 2 | -------------------------------------------------------------------------------- /druid/templates/config/middlemanager/_main.config: -------------------------------------------------------------------------------- 1 | io.druid.cli.Main server middleManager 2 | -------------------------------------------------------------------------------- /druid/charts/utils/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for Kubernetes 3 | name: utils 4 | version: 0.1.0 5 | -------------------------------------------------------------------------------- /druid/charts/utils/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for utils. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | -------------------------------------------------------------------------------- /dockerfiles/metadata-mysql/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mysql:latest 2 | MAINTAINER Jakob Karalus 3 | 4 | EXPOSE 3306 5 | 6 | COPY charset.cnf /etc/mysql/conf.d/charset.cnf 7 | -------------------------------------------------------------------------------- /druid/templates/pivot_cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: pivot-configmap 5 | data: 6 | config.yaml: | 7 | {{ tuple "config/_config.yaml" . | include "template" | indent 4 }} 8 | -------------------------------------------------------------------------------- /druid/templates/config/overlord/_jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xms3g 3 | -Xmx3g 4 | -Duser.timezone=UTC 5 | -Dfile.encoding=UTF-8 6 | -Djava.io.tmpdir=. 7 | -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager 8 | -------------------------------------------------------------------------------- /dockerfiles/metadata-mysql/charset.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | skip-character-set-client-handshake 3 | character-set-server=utf8 4 | collation-server=utf8_general_ci 5 | init-connect = SET NAMES utf8 6 | [client] 7 | default-character-set=utf8 -------------------------------------------------------------------------------- /druid/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for running druid ontop of kubernetes, still work in progress. 3 | name: druid 4 | version: 0.1.0 5 | maintainers: 6 | - name: Jakob K. 7 | email: jakob.karalus@codecentric.de 8 | 9 | -------------------------------------------------------------------------------- /druid/templates/tranquility_cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: metrics-tranquility-configmap 5 | data: 6 | metrics-server.json: | 7 | {{ tuple "config/_metrics-server.json" . | include "template" | indent 4 }} 8 | -------------------------------------------------------------------------------- /druid/templates/config/broker/_jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xms24g 3 | -Xmx24g 4 | -XX:MaxDirectMemorySize=4096m 5 | -Duser.timezone=UTC 6 | -Dfile.encoding=UTF-8 7 | -Djava.io.tmpdir=var/tmp 8 | -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager 9 | -------------------------------------------------------------------------------- /druid/requirements.yaml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: zookeeper 3 | version: 0.1.1 4 | repository: http://storage.googleapis.com/kubernetes-charts-incubator 5 | - name: postgresql 6 | version: 0.2.0 7 | repository: http://storage.googleapis.com/kubernetes-charts -------------------------------------------------------------------------------- /druid/templates/config/historical/_jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xms8g 3 | -Xmx8g 4 | -XX:MaxDirectMemorySize=4096m 5 | -Duser.timezone=UTC 6 | -Dfile.encoding=UTF-8 7 | -Djava.io.tmpdir=var/tmp 8 | -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager 9 | -------------------------------------------------------------------------------- /druid/templates/config/coordinator/_jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xms3g 3 | -Xmx3g 4 | -Duser.timezone=UTC 5 | -Dfile.encoding=UTF-8 6 | -Djava.io.tmpdir=var/tmp 7 | -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager 8 | -Dderby.stream.error.file=var/druid/derby.log 9 | -------------------------------------------------------------------------------- /druid/templates/config/overlord/_runtime.properties: -------------------------------------------------------------------------------- 1 | druid.service=druid/overlord 2 | druid.port=8090 3 | druid.host={{ .Values.nodes.overlord.service.name }} 4 | 5 | druid.indexer.queue.startDelay=PT30S 6 | 7 | druid.indexer.runner.type=remote 8 | druid.indexer.storage.type=metadata 9 | -------------------------------------------------------------------------------- /druid/templates/config/coordinator/_runtime.properties: -------------------------------------------------------------------------------- 1 | druid.service=druid/coordinator 2 | druid.port=8081 3 | druid.host={{ .Values.nodes.coordinator.service.name }} 4 | 5 | 6 | 7 | druid.coordinator.startDelay=PT60S 8 | #Longer Delay for zK 9 | druid.coordinator.period=PT30S 10 | -------------------------------------------------------------------------------- /druid/charts/utils/templates/_include.tpl: -------------------------------------------------------------------------------- 1 | {{- define "template" -}} 2 | {{- $template := index . 0 -}} 3 | {{- $context := index . 1 -}} 4 | {{- $v := $context.Template.Name | split "/" -}} 5 | {{- $last := sub (len $v) 1 | printf "_%d" | index $v -}} 6 | {{- $wtf := printf "%s%s" ($context.Template.Name | trimSuffix $last) $template -}} 7 | {{ include $wtf $context }} 8 | {{- end -}} -------------------------------------------------------------------------------- /druid/templates/config/middlemanager/_jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xms64m 3 | -Xmx64m 4 | -Duser.timezone=UTC 5 | -Dfile.encoding=UTF-8 6 | -Djava.io.tmpdir=var/tmp 7 | -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager 8 | -Ddruid.host=$(echo $HOSTNAME).middlemanager.default.svc.cluster.local 9 | -Ddruid.worker.ip=$(echo $HOSTNAME).middlemanager.default.svc.cluster.local 10 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for running druid ontop of kubernetes, still work in progress. Single Node Druid based of the quickstart container from imply. 3 | Only recommended asquick testing tool. 4 | name: druid-quickstart 5 | version: 0.1.0 6 | maintainers: 7 | - name: Jakob K. 8 | email: jakob.karalus@codecentric.de 9 | 10 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ template "fullname" . }} 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | type: {{ .Values.service.quickstart.type }} 9 | ports: 10 | - port: {{ .Values.service.quickstart.port }} 11 | selector: 12 | app: {{ template "fullname" . }} 13 | -------------------------------------------------------------------------------- /druid/requirements.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: zookeeper 3 | repository: http://storage.googleapis.com/kubernetes-charts-incubator 4 | version: 0.1.1 5 | - name: postgresql 6 | repository: http://storage.googleapis.com/kubernetes-charts 7 | version: 0.2.0 8 | digest: sha256:35059a9b2bc4bb9b830e1cac7c2a332e5e5b57c3b107aea75b19db49ef09aaea 9 | generated: 2016-11-16T10:20:20.593425987+01:00 10 | -------------------------------------------------------------------------------- /druid/charts/utils/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /dockerfiles/overlord/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | 13 | EXPOSE 8090 14 | 15 | 16 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 17 | ENTRYPOINT bin/run-druid overlord conf 18 | -------------------------------------------------------------------------------- /druid/templates/config/broker/_runtime.properties: -------------------------------------------------------------------------------- 1 | druid.service=druid/broker 2 | druid.port=8082 3 | 4 | # HTTP server threads 5 | druid.broker.http.numConnections=5 6 | druid.server.http.numThreads=40 7 | 8 | # Processing threads and buffers 9 | druid.processing.buffer.sizeBytes=536870912 10 | druid.processing.numThreads=7 11 | 12 | # Query cache disabled -- push down caching and merging instead 13 | druid.broker.cache.useCache=false 14 | druid.broker.cache.populateCache=false 15 | -------------------------------------------------------------------------------- /dockerfiles/historical/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: druid8s-historical-deployment 5 | spec: 6 | replicas: 3 7 | template: 8 | metadata: 9 | labels: 10 | app: druid 11 | spec: 12 | containers: 13 | - name: druid-historical 14 | image: jkaralus/druid8s-historical:latest 15 | imagePullPolicy: IfNotPresent 16 | ports: 17 | - containerPort: 8083 18 | -------------------------------------------------------------------------------- /dockerfiles/historical/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | 13 | EXPOSE 8083 14 | 15 | 16 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 17 | ENTRYPOINT cd /iap-druid/imply-$IAP_VERSION/ && bin/run-druid historical conf 18 | -------------------------------------------------------------------------------- /dockerfiles/tranquility/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | 13 | EXPOSE 8200 14 | 15 | 16 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 17 | ENTRYPOINT bin/tranquility server -configFile conf/tranquility/metrics-server.json 18 | -------------------------------------------------------------------------------- /dockerfiles/broker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | 13 | EXPOSE 8082 14 | 15 | #TODO expose only Ports needed. 16 | 17 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 18 | ENTRYPOINT cd /iap-druid/imply-$IAP_VERSION/ && bin/run-druid broker conf 19 | -------------------------------------------------------------------------------- /dockerfiles/pivot/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | EXPOSE 9095 13 | 14 | #ADD config.yaml /iap-druid/imply-$IAP_VERSION/conf/pivot/config.yaml 15 | 16 | 17 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 18 | ENTRYPOINT bin/run-pivot conf 19 | #ENTRYPOINT /bin/bash 20 | -------------------------------------------------------------------------------- /druid/templates/secrets.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ template "fullname" . }} 5 | labels: 6 | app: {{ template "fullname" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: "{{ .Release.Name }}" 9 | heritage: "{{ .Release.Service }}" 10 | type: Opaque 11 | data: 12 | {{ if .Values.postgresql.postgresPassword }} 13 | mysql-password: {{ .Values.postgresql.postgresPassword | b64enc | quote }} 14 | {{ else }} 15 | mysql-password: {{ randAlphaNum 10 | b64enc | quote }} 16 | {{ end }} -------------------------------------------------------------------------------- /druid/charts/utils/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 24 -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- $name := default .Chart.Name .Values.nameOverride -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 16 | {{- end -}} 17 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 24 -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- $name := default .Chart.Name .Values.nameOverride -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 16 | {{- end -}} 17 | -------------------------------------------------------------------------------- /dockerfiles/local-zk/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | #TODO needed? complete Solo zK without Druid 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | 13 | EXPOSE 2181 2888 3888 14 | 15 | #TODO expose only Ports needed. 16 | 17 | 18 | 19 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 20 | ENTRYPOINT cd /iap-druid/imply-$IAP_VERSION/ && bin/run-zk conf 21 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/manifests/druid-historical-rc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: druid-historical-rc 5 | labels: 6 | heritage: helm 7 | app: druid 8 | spec: 9 | replicas: 1 10 | selector: 11 | name: druid-historical 12 | template: 13 | metadata: 14 | labels: 15 | name: druid-historical 16 | app: druid 17 | spec: 18 | containers: 19 | - name: druid-historical 20 | image: jkaralus/druid8s-historical:latest 21 | imagePullPolicy: IfNotPresent 22 | ports: 23 | - containerPort: 8083 -------------------------------------------------------------------------------- /druid/templates/config/historical/_runtime.properties: -------------------------------------------------------------------------------- 1 | druid.service=druid/historical 2 | druid.port=8083 3 | 4 | # HTTP server threads 5 | druid.server.http.numThreads=40 6 | 7 | # Processing threads and buffers 8 | druid.processing.buffer.sizeBytes=536870912 9 | druid.processing.numThreads=7 10 | 11 | # Segment storage 12 | druid.segmentCache.locations=[{"path":"/var/druid/segment-cache","maxSize"\:130000000000}] 13 | druid.server.maxSize=130000000000 14 | 15 | # Query cache 16 | druid.historical.cache.useCache=true 17 | druid.historical.cache.populateCache=true 18 | druid.cache.type=local 19 | druid.cache.sizeInBytes=2000000000 20 | -------------------------------------------------------------------------------- /dockerfiles/coordinator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | EXPOSE 8081 13 | EXPOSE 8082 14 | EXPOSE 8083 15 | EXPOSE 8090 16 | EXPOSE 8091 17 | EXPOSE 3306 18 | EXPOSE 2181 2888 3888 19 | EXPOSE 8100 8101 8102 8103 8104 20 | #TODO expose only Ports needed. 21 | 22 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 23 | #ENTRYPOINT bash 24 | ENTRYPOINT bin/run-druid coordinator conf 25 | -------------------------------------------------------------------------------- /dockerfiles/middleManager/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jkaralus/druid8s-base:latest 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | EXPOSE 8081 13 | EXPOSE 8082 14 | EXPOSE 8083 15 | EXPOSE 8090 16 | EXPOSE 8091 17 | EXPOSE 3306 18 | EXPOSE 2181 2888 3888 19 | EXPOSE 8100 8101 8102 8103 8104 20 | #TODO expose only Ports needed. 21 | 22 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 23 | ENTRYPOINT cd /iap-druid/imply-$IAP_VERSION/ && bin/run-druid middleManager conf 24 | -------------------------------------------------------------------------------- /dockerfiles/pivot/config.yaml: -------------------------------------------------------------------------------- 1 | # The port on which the Pivot server will listen on. 2 | port: 9095 3 | 4 | clusters: 5 | - name: druid 6 | type: druid 7 | 8 | # A Druid broker node that can serve data (only used if you have Druid based data source) 9 | host: druid-broker-service:8082 10 | 11 | # Background introspection frequency (in ms) 12 | # - Set to '0' to disable background introspection. 13 | sourceListRefreshInterval: 30000 14 | 15 | # Foreground introspection 16 | # Checks for new dataSources every time Pivot is loaded (default: false) 17 | sourceListRefreshOnLoad: false 18 | 19 | # Data source definitions will go here 20 | dataSources: [] 21 | -------------------------------------------------------------------------------- /druid/templates/volume-claim.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: {{ .Values.druidSegmentDeepStorage }} 5 | annotations: 6 | volume.alpha.kubernetes.io/storage-class: "anything" 7 | #TODO should be a EBS/NFS or similiar mount in a cluster enviroment 8 | spec: 9 | accessModes: 10 | - ReadWriteOnce 11 | resources: 12 | requests: 13 | storage: 1Gi 14 | --- 15 | kind: PersistentVolumeClaim 16 | apiVersion: v1 17 | metadata: 18 | name: {{ .Values.druidSegmentCache }} 19 | annotations: 20 | volume.alpha.kubernetes.io/storage-class: "HostPath" 21 | spec: 22 | accessModes: 23 | - ReadWriteOnce 24 | resources: 25 | requests: 26 | storage: 1Gi -------------------------------------------------------------------------------- /dockerfiles/build_images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | TAG="latest" 5 | docker build -t jkaralus/druid8s-base:$TAG base 6 | docker build -t jkaralus/druid8s-mysql-utf metadata-mysql 7 | docker build -t jkaralus/druid8s-quickstart:$TAG quickstart 8 | docker build -t jkaralus/druid8s-local-zk:$TAG local-zk 9 | docker build -t jkaralus/druid8s-coordinator:$TAG coordinator 10 | docker build -t jkaralus/druid8s-overlord:$TAG overlord 11 | docker build -t jkaralus/druid8s-middlemanager:$TAG middleManager 12 | docker build -t jkaralus/druid8s-historical:$TAG historical 13 | docker build -t jkaralus/druid8s-broker:$TAG broker 14 | docker build -t jkaralus/druid8s-pivot:$TAG pivot 15 | docker build -t jkaralus/druid8s-tranquility:$TAG tranquility -------------------------------------------------------------------------------- /druid/templates/config/_config.yaml: -------------------------------------------------------------------------------- 1 | # The port on which the Pivot server will listen on. 2 | port: 9095 3 | 4 | # Pivot runtime directory TODO set to better path 5 | varDir: . 6 | 7 | clusters: 8 | - name: druid 9 | type: druid 10 | 11 | # A Druid broker node that can serve data (only used if you have Druid based data source) 12 | host: {{ .Values.nodes.broker.service.name }}:8082 13 | 14 | # Background introspection frequency (in ms) 15 | # - Set to '0' to disable background introspection. 16 | sourceListRefreshInterval: 30000 17 | 18 | # Foreground introspection 19 | # Checks for new dataSources every time Pivot is loaded (default: false) 20 | sourceListRefreshOnLoad: false 21 | 22 | # Data source definitions will go here 23 | dataSources: [] 24 | -------------------------------------------------------------------------------- /dockerfiles/base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | MAINTAINER Jakob K. jakob.karalus@gmx.net 4 | 5 | ENV DRUID_VERSION 0.7.1.1 6 | ENV IAP_VERSION 2.0.0 7 | 8 | RUN apt-get update && apt-get install -y wget 9 | RUN wget -q -O - https://deb.nodesource.com/setup_4.x | bash - 10 | RUN apt-get update && apt-get install -y openjdk-8-jre nodejs 11 | RUN apt-get install -y libthreads-perl libc6 perl perl-doc python 12 | 13 | RUN mkdir /iap-druid 14 | RUN wget -q -O - https://static.imply.io/release/imply-$IAP_VERSION.tar.gz | tar -xzf - -C /iap-druid/ 15 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 16 | 17 | #ADD common.runtime.properties /iap-druid/imply-$IAP_VERSION/conf/druid/_common/common.runtime.properties 18 | ADD run-druid /iap-druid/imply-$IAP_VERSION/bin/run-druid 19 | 20 | 21 | ENTRYPOINT bash 22 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: {{ template "fullname" . }} 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | app: {{ template "fullname" . }} 13 | spec: 14 | containers: 15 | - name: {{ .Chart.Name }} 16 | image: "{{ .Values.image.quickstart.repository }}:{{ .Values.image.quickstart.tag }}" 17 | imagePullPolicy: {{ .Values.image.quickstart.pullPolicy }} 18 | ports: 19 | - containerPort: {{ .Values.service.quickstart.port }} 20 | #TODO add readiness & liveness probes 21 | resources: 22 | {{ toYaml .Values.resources | indent 12 }} 23 | -------------------------------------------------------------------------------- /dockerfiles/base/run-druid: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | if [ "$#" -gt 2 ] 4 | then 5 | echo "usage: $0 [conf-dir]" >&2 6 | exit 1 7 | fi 8 | 9 | PWD="$(pwd)" 10 | WHEREAMI="$(dirname "$0")" 11 | WHATAMI="$1" 12 | 13 | if [ "$#" -eq 1 ] 14 | then 15 | CONFDIR="$WHEREAMI/../conf" 16 | else 17 | CONFDIR="$2" 18 | fi 19 | 20 | CONFDIR="$(cd "$CONFDIR" && pwd)/druid" 21 | WHEREAMI="$(cd "$WHEREAMI" && pwd)" 22 | 23 | cd "$WHEREAMI/.." 24 | exec java `cat "$CONFDIR"/"$WHATAMI"/jvm.config | while read line; do echo $(eval echo $(echo $line)); done | xargs` \ 25 | -cp "$CONFDIR"/_common:"$CONFDIR"/"$WHATAMI":"dist/druid/lib/*" \ 26 | `cat "$CONFDIR"/$WHATAMI/main.config | xargs` 27 | 28 | exec java `cat conf/druid/broker/jvm.config | xargs` -cp conf/druid/_common:conf/broker:"dist/druid/lib/*" `cat conf/druid/broker/main.config | xargs` -------------------------------------------------------------------------------- /dockerfiles/pivot/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-pivot-pod 5 | labels: 6 | name: druid-pivot-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-pivot 12 | image: jkaralus/druid8s-pivot:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 9095 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-pivot-service 22 | labels: 23 | name: druid-pivot-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 9095 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-pivot-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/broker/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-broker-pod 5 | labels: 6 | name: druid-broker-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-broker 12 | image: jkaralus/druid8s-broker:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 8082 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-broker-service 22 | labels: 23 | name: druid-broker-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 8082 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-broker-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/overlord/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-overlord-pod 5 | labels: 6 | name: druid-overlord-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-overlord 12 | image: jkaralus/druid8s-overlord:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 8090 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-overlord-service 22 | labels: 23 | name: druid-overlord-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 8090 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-overlord-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/quickstart/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-quckstart-pod 5 | labels: 6 | name: druid-quickstart-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-quickstart 12 | image: jkaralus/druid8s-quickstart:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 9095 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-quickstart-service 22 | labels: 23 | name: druid-quickstart-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 9095 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-quickstart-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/historical/Single_Pod.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-historical-pod 5 | labels: 6 | name: druid-historical-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-historical 12 | image: jkaralus/druid8s-historical:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 8083 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-historical-service 22 | labels: 23 | name: druid-historical-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 8083 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-historical-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/coordinator/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-coordinator-pod 5 | labels: 6 | name: druid-coordinator-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-coordinator 12 | image: jkaralus/druid8s-coordinator:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 8081 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-coordinator-service 22 | labels: 23 | name: druid-coordinator-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 8081 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-coordinator-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for druid. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | replicaCount: 1 5 | 6 | #TODO remove quickstart stuff 7 | 8 | image: 9 | historical: 10 | repository: jkaralus/druid8s-historical 11 | tag: latest 12 | pullPolicy: IfNotPresent 13 | quickstart: 14 | repository: jkaralus/druid8s-quickstart 15 | tag: latest 16 | pullPolicy: IfNotPresent 17 | service: 18 | historical: 19 | name: historical 20 | type: ClusterIP 21 | externalPort: 8083 22 | internalPort: 8083 23 | quickstart: 24 | name: quickstart 25 | type: NodePort 26 | port: 9095 27 | #TODO remove ulgy hotfix 28 | type: NodePort 29 | 30 | resources: 31 | limits: 32 | cpu: 100m 33 | memory: 128Mi 34 | requests: 35 | cpu: 100m 36 | memory: 128Mi 37 | 38 | -------------------------------------------------------------------------------- /druid/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 24 -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- $name := default .Chart.Name .Values.nameOverride -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 16 | {{- end -}} 17 | 18 | {{/* 19 | Creates a truncated 24char name with the ReleaseName and a given Parameter, useful for sub-charts where fullname is used 20 | on the parent char overrides the name. 21 | */}} 22 | {{- define "fullCustomName" -}} 23 | {{- $name := index . 0 -}} 24 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 25 | {{- end -}} -------------------------------------------------------------------------------- /dockerfiles/middleManager/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-middlemanager-pod 5 | labels: 6 | name: druid-middlemanager-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-middlemanager 12 | image: jkaralus/druid8s-middlemanager:latest 13 | imagePullPolicy: IfNotPresent 14 | ports: 15 | - 16 | containerPort: 8091 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-middlemanager-service 22 | labels: 23 | name: druid-middlemanager-pod 24 | context: druid-k8s 25 | spec: 26 | type: NodePort 27 | ports: 28 | # the port that this service should serve on 29 | - port: 8091 30 | # label keys and values that must match in order to receive traffic for this service 31 | selector: 32 | name: druid-middlemanager-pod 33 | context: druid-k8s 34 | -------------------------------------------------------------------------------- /dockerfiles/local-zk/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-local-zk-pod 5 | labels: 6 | name: druid-local-zk-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - name: druid-local-zk 11 | image: jkaralus/druid8s-local-zk:latest 12 | imagePullPolicy: IfNotPresent 13 | ports: 14 | - containerPort: 2181 15 | - containerPort: 2888 16 | - containerPort: 3888 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: druid-local-zk-service 22 | labels: 23 | name: druid-local-zk-pod 24 | context: druid-k8s 25 | spec: 26 | ports: 27 | # the port that this service should serve on 28 | - name: client 29 | port: 2181 30 | - name: follower 31 | port: 2888 32 | - name: election 33 | port: 3888 34 | selector: 35 | name: druid-local-zk-pod 36 | context: druid-k8s 37 | -------------------------------------------------------------------------------- /druid/templates/config/middlemanager/_runtime.properties: -------------------------------------------------------------------------------- 1 | druid.service=druid/middlemanager 2 | druid.port=8091 3 | #druid.host={{ .Values.nodes.middlemanager.service.name }} 4 | 5 | # Number of tasks per middleManager 6 | druid.worker.capacity=2 7 | 8 | # Task launch parameters 9 | druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager 10 | druid.indexer.task.baseTaskDir=var/druid/task 11 | druid.indexer.task.restoreTasksOnRestart=true 12 | 13 | # HTTP server threads 14 | druid.server.http.numThreads=40 15 | 16 | # Processing threads and buffers 17 | druid.processing.buffer.sizeBytes=536870912 18 | druid.processing.numThreads=2 19 | 20 | # Hadoop indexing 21 | druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp 22 | druid.indexer.task.defaultHadoopCoordinates=["org.apache.hadoop:hadoop-client:2.3.0"] 23 | 24 | 25 | #druid.worker.ip=druid-middlemanager-service 26 | 27 | #Defined in jvm config 28 | #druid.worker.host=$HOSTNAME -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff: 7 | .idea/workspace.xml 8 | .idea/tasks.xml 9 | .idea/dictionaries 10 | .idea/vcs.xml 11 | .idea/jsLibraryMappings.xml 12 | 13 | # Sensitive or high-churn files: 14 | .idea/dataSources.ids 15 | .idea/dataSources.xml 16 | .idea/dataSources.local.xml 17 | .idea/sqlDataSources.xml 18 | .idea/dynamic.xml 19 | .idea/uiDesigner.xml 20 | 21 | # Gradle: 22 | .idea/gradle.xml 23 | .idea/libraries 24 | 25 | # Mongo Explorer plugin: 26 | .idea/mongoSettings.xml 27 | 28 | ## File-based project format: 29 | *.iws 30 | 31 | ## Plugin-specific files: 32 | 33 | # IntelliJ 34 | /out/ 35 | 36 | # mpeltonen/sbt-idea plugin 37 | .idea_modules/ 38 | 39 | # JIRA plugin 40 | atlassian-ide-plugin.xml 41 | 42 | # Crashlytics plugin (for Android Studio and IntelliJ) 43 | com_crashlytics_export_strings.xml 44 | crashlytics.properties 45 | crashlytics-build.properties 46 | fabric.properties 47 | 48 | *.tgz -------------------------------------------------------------------------------- /dockerfiles/quickstart/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | # Expose ports: 4 | # - 8081: HTTP (coordinator) 5 | # - 8082: HTTP (broker) 6 | # - 8083: HTTP (historical) 7 | # - 8090: HTTP (overlord) 8 | # - 8091: HTTP (middlemanager) 9 | # - 3306: MySQL 10 | # - 2181 2888 3888: ZooKeeper 11 | # - 8100 8101 8102 8103 8104 : peon ports 12 | 13 | ENV DRUID_VERSION 0.7.1.1 14 | ENV IAP_VERSION 1.3.0 15 | 16 | RUN apt-get update && apt-get install -y wget 17 | RUN wget -q -O - https://deb.nodesource.com/setup_4.x | bash - 18 | RUN apt-get update && apt-get install -y openjdk-8-jre nodejs 19 | RUN apt-get install -y libthreads-perl libc6 perl perl-doc python 20 | 21 | RUN mkdir /iap-druid 22 | RUN wget -q -O - https://static.imply.io/release/imply-$IAP_VERSION.tar.gz | tar -xzf - -C /iap-druid/ 23 | 24 | EXPOSE 8081 25 | EXPOSE 8082 26 | EXPOSE 8083 27 | EXPOSE 8090 28 | EXPOSE 8091 29 | EXPOSE 3306 30 | EXPOSE 2181 2888 3888 31 | EXPOSE 8100 8101 8102 8103 8104 32 | ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8 LC_ALL=C.UTF-8 LANGUAGE=en_US.UTF-8 33 | 34 | 35 | 36 | WORKDIR /iap-druid/imply-$IAP_VERSION/ 37 | ENTRYPOINT cd /iap-druid/imply-$IAP_VERSION/ && bin/supervise -c conf/supervise/quickstart.conf 38 | -------------------------------------------------------------------------------- /druid/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if contains "NodePort" .Values.service.type }} 3 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "fullname" . }}) 4 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 5 | echo http://$NODE_IP:$NODE_PORT/login 6 | {{- else if contains "LoadBalancer" .Values.service.type }} 7 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 8 | You can watch the status of by running 'kubectl get svc -w {{ template "fullname" . }}' 9 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 10 | echo http://$SERVICE_IP:{{ .Values.Master.ServicePort }} 11 | {{- else if contains "ClusterIP" .Values.service.type }} 12 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "fullname" . }}" -o jsonpath="{.items[0].metadata.name}") 13 | echo "Visit http://127.0.0.1:8080 to use your application" 14 | kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }} 15 | {{- end }} 16 | 17 | -------------------------------------------------------------------------------- /dockerfiles/druid-quickstart/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if contains "NodePort" .Values.service.type }} 3 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "fullname" . }}) 4 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 5 | echo http://$NODE_IP:$NODE_PORT/login 6 | {{- else if contains "LoadBalancer" .Values.service.type }} 7 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 8 | You can watch the status of by running 'kubectl get svc -w {{ template "fullname" . }}' 9 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 10 | echo http://$SERVICE_IP:{{ .Values.Master.ServicePort }} 11 | {{- else if contains "ClusterIP" .Values.service.type }} 12 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "fullname" . }}" -o jsonpath="{.items[0].metadata.name}") 13 | echo "Visit http://127.0.0.1:8080 to use your application" 14 | kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }} 15 | {{- end }} 16 | -------------------------------------------------------------------------------- /dockerfiles/metadata-mysql/Deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: druid-metadata-mysql-pod 5 | labels: 6 | name: druid-metadata-mysql-pod 7 | context: druid-k8s 8 | spec: 9 | containers: 10 | - 11 | name: druid-metadata-mysql 12 | image: jkaralus/druid8s-mysql-utf 13 | imagePullPolicy: IfNotPresent 14 | env: 15 | - name: MYSQL_DATABASE 16 | value: "druid" 17 | #create druid database 18 | - name: MYSQL_ROOT_PASSWORD 19 | value: "supersecretrootpassword123456789!@" 20 | #TODO use secrets for that 21 | - name: MYSQL_USER 22 | value: "druid" 23 | #TODO better Handling, secret 24 | - name: MYSQL_PASSWORD 25 | value: "secret" 26 | #TODO better Handling, secret 27 | ports: 28 | - 29 | containerPort: 3306 30 | --- 31 | apiVersion: v1 32 | kind: Service 33 | metadata: 34 | name: druid-metadata-mysql-service 35 | labels: 36 | name: druid-metadata-mysql-pod 37 | context: druid-k8s 38 | spec: 39 | type: NodePort 40 | ports: 41 | # the port that this service should serve on 42 | - port: 3306 43 | # label keys and values that must match in order to receive traffic for this service 44 | selector: 45 | name: druid-metadata-mysql-pod 46 | context: druid-k8s 47 | -------------------------------------------------------------------------------- /druid/templates/pivot.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: "{{ .Chart.Name }}-{{ .Values.nodes.pivot.name }}-deployment" 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.nodes.pivot.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | name: "{{ .Chart.Name }}-{{ .Values.nodes.pivot.name }}-rc" 13 | app: {{ template "fullname" . }} 14 | spec: 15 | containers: 16 | - name: "{{ .Chart.Name }}-{{ .Values.nodes.pivot.name }}" 17 | image: "{{ .Values.nodes.pivot.image.repository }}:{{ .Values.nodes.pivot.image.tag }}" 18 | imagePullPolicy: {{ .Values.nodes.pivot.image.pullPolicy }} 19 | ports: 20 | {{range .Values.nodes.pivot.service.ports }} 21 | - containerPort: {{ .port }} 22 | {{end}} 23 | volumeMounts: 24 | - name: config-volume 25 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/pivot/" 26 | #TODO add readiness & liveness probes 27 | volumes: 28 | - name: config-volume 29 | configMap: 30 | name: pivot-configmap 31 | items: 32 | - key: config.yaml 33 | path: config.yaml 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: {{ .Values.nodes.pivot.service.name }} 39 | #TODO flexible service Name, configMap 40 | labels: 41 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 42 | spec: 43 | type: {{ .Values.nodes.pivot.service.type }} 44 | ports: 45 | {{range .Values.nodes.pivot.service.ports }} 46 | - name: {{ .name }} 47 | port: {{ .port }} 48 | {{end}} 49 | selector: 50 | name: "{{ .Chart.Name }}-{{ .Values.nodes.pivot.name }}-rc" -------------------------------------------------------------------------------- /druid/templates/tranquility.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: "{{ .Chart.Name }}-{{ .Values.nodes.tranquility.name }}-deployment" 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.nodes.tranquility.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | name: "{{ .Chart.Name }}-{{ .Values.nodes.tranquility.name }}-rc" 13 | app: {{ template "fullname" . }} 14 | spec: 15 | containers: 16 | - name: "{{ .Chart.Name }}-{{ .Values.nodes.tranquility.name }}" 17 | image: "{{ .Values.nodes.tranquility.image.repository }}:{{ .Values.nodes.tranquility.image.tag }}" 18 | imagePullPolicy: {{ .Values.nodes.tranquility.image.pullPolicy }} 19 | ports: 20 | {{range .Values.nodes.tranquility.service.ports }} 21 | - containerPort: {{ .port }} 22 | {{end}} 23 | volumeMounts: 24 | - name: config-volume 25 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/tranquility/" 26 | #TODO add readiness & liveness probes 27 | volumes: 28 | - name: config-volume 29 | configMap: 30 | name: metrics-tranquility-configmap 31 | items: 32 | - key: metrics-server.json 33 | path: metrics-server.json 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: {{ .Values.nodes.tranquility.service.name }} 39 | #TODO flexible service Name, configMap 40 | labels: 41 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 42 | spec: 43 | type: {{ .Values.nodes.tranquility.service.type }} 44 | ports: 45 | {{range .Values.nodes.tranquility.service.ports }} 46 | - name: {{ .name }} 47 | port: {{ .port }} 48 | {{end}} 49 | selector: 50 | name: "{{ .Chart.Name }}-{{ .Values.nodes.tranquility.name }}-rc" -------------------------------------------------------------------------------- /druid/templates/broker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: "{{ .Chart.Name }}-{{ .Values.nodes.broker.name }}-deployment" 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.nodes.broker.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | name: "{{ .Chart.Name }}-{{ .Values.nodes.broker.name }}-rc" 13 | app: {{ template "fullname" . }} 14 | spec: 15 | hostname: broker 16 | subdomain: druid 17 | containers: 18 | - name: "{{ .Chart.Name }}-{{ .Values.nodes.broker.name }}" 19 | image: "{{ .Values.nodes.broker.image.repository }}:{{ .Values.nodes.broker.image.tag }}" 20 | imagePullPolicy: {{ .Values.nodes.broker.image.pullPolicy }} 21 | volumeMounts: 22 | - name: config-volume 23 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/druid/_common/" 24 | 25 | #TODO add readiness & liveness probes 26 | #TODO reenable ressources here 27 | ports: 28 | {{range .Values.nodes.broker.service.ports }} 29 | - containerPort: {{ .port }} 30 | {{end}} 31 | 32 | volumes: 33 | - name: config-volume 34 | configMap: 35 | name: common-runtime-configmap 36 | items: 37 | - key: common.runtime.properties 38 | path: common.runtime.properties 39 | --- 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | name: {{ .Values.nodes.broker.service.name }} 44 | #TODO flexible service Name, configMap 45 | labels: 46 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 47 | spec: 48 | type: {{ .Values.nodes.broker.service.type }} 49 | ports: 50 | {{range .Values.nodes.broker.service.ports }} 51 | - name: {{ .name }} 52 | port: {{ .port }} 53 | {{end}} 54 | selector: 55 | name: "{{ .Chart.Name }}-{{ .Values.nodes.broker.name }}-rc" -------------------------------------------------------------------------------- /druid/templates/common_cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: common-runtime-configmap 5 | data: 6 | common.runtime.properties: | 7 | {{ tuple "config/_common-runtime.properties" . | include "template" | indent 4 }} 8 | overlord.runtime.properties: | 9 | {{ tuple "config/overlord/_runtime.properties" . | include "template" | indent 4 }} 10 | overlord.jvm.config: | 11 | {{ tuple "config/overlord/_jvm.config" . | include "template" | indent 4 }} 12 | overlord.main.config: | 13 | {{ tuple "config/overlord/_main.config" . | include "template" | indent 4 }} 14 | middlemanager.runtime.properties: | 15 | {{ tuple "config/middlemanager/_runtime.properties" . | include "template" | indent 4 }} 16 | middlemanager.jvm.config: | 17 | {{ tuple "config/middlemanager/_jvm.config" . | include "template" | indent 4 }} 18 | middlemanager.main.config: | 19 | {{ tuple "config/middlemanager/_main.config" . | include "template" | indent 4 }} 20 | coordinator.runtime.properties: | 21 | {{ tuple "config/coordinator/_runtime.properties" . | include "template" | indent 4 }} 22 | coordinator.jvm.config: | 23 | {{ tuple "config/coordinator/_jvm.config" . | include "template" | indent 4 }} 24 | coordinator.main.config: | 25 | {{ tuple "config/coordinator/_main.config" . | include "template" | indent 4 }} 26 | historical.runtime.properties: | 27 | {{ tuple "config/historical/_runtime.properties" . | include "template" | indent 4 }} 28 | historical.jvm.config: | 29 | {{ tuple "config/historical/_jvm.config" . | include "template" | indent 4 }} 30 | historical.main.config: | 31 | {{ tuple "config/historical/_main.config" . | include "template" | indent 4 }} 32 | broker.runtime.properties: | 33 | {{ tuple "config/broker/_runtime.properties" . | include "template" | indent 4 }} 34 | broker.jvm.config: | 35 | {{ tuple "config/broker/_jvm.config" . | include "template" | indent 4 }} 36 | broker.main.config: | 37 | {{ tuple "config/broker/_main.config" . | include "template" | indent 4 }} -------------------------------------------------------------------------------- /druid/templates/middlemanager.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ .Values.nodes.middlemanager.service.name }} 6 | labels: 7 | app: druid 8 | spec: 9 | type: {{ .Values.nodes.middlemanager.service.type }} 10 | ports: 11 | {{range .Values.nodes.middlemanager.service.ports }} 12 | - name: {{ .name }} 13 | port: {{ .port }} 14 | {{end}} 15 | # *.nginx.default.svc.cluster.local 16 | selector: 17 | name: middlemanager 18 | --- 19 | apiVersion: apps/v1alpha1 20 | kind: PetSet 21 | metadata: 22 | name: {{ .Values.nodes.middlemanager.service.name }} 23 | spec: 24 | serviceName: {{ .Values.nodes.middlemanager.service.name }} 25 | replicas: 2 26 | template: 27 | metadata: 28 | labels: 29 | app: druid 30 | annotations: 31 | pod.alpha.kubernetes.io/initialized: "true" 32 | spec: 33 | terminationGracePeriodSeconds: 0 34 | containers: 35 | - name: {{ .Values.nodes.middlemanager.service.name }} 36 | image: "{{ .Values.nodes.middlemanager.image.repository }}:{{ .Values.nodes.middlemanager.image.tag }}" 37 | imagePullPolicy: {{ .Values.nodes.middlemanager.image.pullPolicy }} 38 | ports: 39 | {{range .Values.nodes.middlemanager.service.ports }} 40 | - containerPort: {{ .port }} 41 | {{end}} 42 | volumeMounts: 43 | - name: config-volume 44 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/druid/" 45 | volumes: 46 | - name: config-volume 47 | configMap: 48 | name: common-runtime-configmap 49 | items: 50 | - key: common.runtime.properties 51 | path: _common/common.runtime.properties 52 | - key: "{{ .Values.nodes.middlemanager.name }}.runtime.properties" 53 | path: middleManager/runtime.properties 54 | - key: "{{ .Values.nodes.middlemanager.name }}.jvm.config" 55 | path: middleManager/jvm.config 56 | - key: "{{ .Values.nodes.middlemanager.name }}.main.config" 57 | path: middleManager/main.config -------------------------------------------------------------------------------- /druid/templates/overlord.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: "{{ .Chart.Name }}-{{ .Values.nodes.overlord.name }}-deployment" 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.nodes.overlord.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | name: "{{ .Chart.Name }}-{{ .Values.nodes.overlord.name }}-rc" 13 | app: {{ template "fullname" . }} 14 | spec: 15 | containers: 16 | - name: "{{ .Chart.Name }}-{{ .Values.nodes.overlord.name }}" 17 | image: "{{ .Values.nodes.overlord.image.repository }}:{{ .Values.nodes.overlord.image.tag }}" 18 | imagePullPolicy: {{ .Values.nodes.overlord.image.pullPolicy }} 19 | ports: 20 | {{range .Values.nodes.overlord.service.ports }} 21 | - containerPort: {{ .port }} 22 | {{end}} 23 | volumeMounts: 24 | - name: config-volume 25 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/druid/" 26 | #TODO add readiness & liveness probes 27 | volumes: 28 | - name: config-volume 29 | configMap: 30 | name: common-runtime-configmap 31 | items: 32 | - key: common.runtime.properties 33 | path: _common/common.runtime.properties 34 | - key: "{{ .Values.nodes.overlord.name }}.runtime.properties" 35 | path: overlord/runtime.properties 36 | - key: "{{ .Values.nodes.overlord.name }}.jvm.config" 37 | path: overlord/jvm.config 38 | - key: "{{ .Values.nodes.overlord.name }}.main.config" 39 | path: overlord/main.config 40 | --- 41 | apiVersion: v1 42 | kind: Service 43 | metadata: 44 | name: {{ .Values.nodes.overlord.service.name }} 45 | #TODO flexible service Name, configMap 46 | labels: 47 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 48 | spec: 49 | type: {{ .Values.nodes.overlord.service.type }} 50 | ports: 51 | {{range .Values.nodes.overlord.service.ports }} 52 | - name: {{ .name }} 53 | port: {{ .port }} 54 | {{end}} 55 | selector: 56 | name: "{{ .Chart.Name }}-{{ .Values.nodes.overlord.name }}-rc" -------------------------------------------------------------------------------- /druid/templates/coordinator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: "{{ .Chart.Name }}-{{ .Values.nodes.coordinator.name }}-deployment" 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.nodes.coordinator.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | name: "{{ .Chart.Name }}-{{ .Values.nodes.coordinator.name }}-rc" 13 | app: {{ template "fullname" . }} 14 | spec: 15 | containers: 16 | - name: "{{ .Chart.Name }}-{{ .Values.nodes.coordinator.name }}" 17 | image: "{{ .Values.nodes.coordinator.image.repository }}:{{ .Values.nodes.coordinator.image.tag }}" 18 | imagePullPolicy: {{ .Values.nodes.coordinator.image.pullPolicy }} 19 | ports: 20 | {{range .Values.nodes.coordinator.service.ports }} 21 | - containerPort: {{ .port }} 22 | {{end}} 23 | volumeMounts: 24 | - name: config-volume 25 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/druid/" 26 | 27 | #TODO add readiness & liveness probes 28 | volumes: 29 | - name: config-volume 30 | configMap: 31 | name: common-runtime-configmap 32 | items: 33 | - key: common.runtime.properties 34 | path: _common/common.runtime.properties 35 | - key: "{{ .Values.nodes.coordinator.name }}.runtime.properties" 36 | path: coordinator/runtime.properties 37 | - key: "{{ .Values.nodes.coordinator.name }}.jvm.config" 38 | path: coordinator/jvm.config 39 | - key: "{{ .Values.nodes.coordinator.name }}.main.config" 40 | path: coordinator/main.config 41 | --- 42 | apiVersion: v1 43 | kind: Service 44 | metadata: 45 | name: {{ .Values.nodes.coordinator.service.name }} 46 | #TODO flexible service Name, configMap 47 | labels: 48 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 49 | spec: 50 | type: {{ .Values.nodes.coordinator.service.type }} 51 | ports: 52 | {{range .Values.nodes.coordinator.service.ports }} 53 | - name: {{ .name }} 54 | port: {{ .port }} 55 | {{end}} 56 | selector: 57 | name: "{{ .Chart.Name }}-{{ .Values.nodes.coordinator.name }}-rc" 58 | -------------------------------------------------------------------------------- /druid/templates/config/_metrics-server.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataSources" : [ 3 | { 4 | "spec" : { 5 | "dataSchema" : { 6 | "dataSource" : "metrics", 7 | "parser" : { 8 | "type" : "string", 9 | "parseSpec" : { 10 | "timestampSpec" : { 11 | "column" : "timestamp", 12 | "format" : "auto" 13 | }, 14 | "dimensionsSpec" : { 15 | "dimensions" : [], 16 | "dimensionExclusions" : [ 17 | "timestamp", 18 | "value" 19 | ] 20 | }, 21 | "format" : "json" 22 | } 23 | }, 24 | "granularitySpec" : { 25 | "type" : "uniform", 26 | "segmentGranularity" : "hour", 27 | "queryGranularity" : "none" 28 | }, 29 | "metricsSpec" : [ 30 | { 31 | "type" : "count", 32 | "name" : "count" 33 | }, 34 | { 35 | "name" : "value_sum", 36 | "type" : "doubleSum", 37 | "fieldName" : "value" 38 | }, 39 | { 40 | "fieldName" : "value", 41 | "name" : "value_min", 42 | "type" : "doubleMin" 43 | }, 44 | { 45 | "type" : "doubleMax", 46 | "name" : "value_max", 47 | "fieldName" : "value" 48 | } 49 | ] 50 | }, 51 | "ioConfig" : { 52 | "type" : "realtime" 53 | }, 54 | "tuningConfig" : { 55 | "type" : "realtime", 56 | "maxRowsInMemory" : "100000", 57 | "intermediatePersistPeriod" : "PT2M", 58 | "windowPeriod" : "PT2M" 59 | } 60 | }, 61 | "properties" : { 62 | "task.partitions" : "1", 63 | "task.replicants" : "1" 64 | } 65 | } 66 | ], 67 | "properties" : { 68 | "zookeeper.connect" : "{{ printf "%s-%s" .Release.Name "zk" | trunc 24 }}", 69 | "druid.discovery.curator.path" : "/druid/discovery", 70 | "druid.selectors.indexing.serviceName" : "druid/overlord", 71 | "http.port" : "8200", 72 | "http.threads" : "40", 73 | "serialization.format" : "smile", 74 | "druidBeam.taskLocator": "overlord" 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Work in Progress. 2 | 3 | # druid-kubernetes 4 | Setup Druid on a Kubernetes Cluster with Helm. 5 | The Setup of zookeeper and postgres (as metadata storage is handled by subcharts), and therefore zK run with a PetSet in a 3 Node HA. 6 | 7 | All Druid Nodes run in their own pods, which are created by deployments. Should be somewhat scaleable. 8 | 9 | SegmentCache is persistentVolume with `HostPath` so historicals on the same node could be using the same segment cache. SegmentStorage is currently set to NFS. 10 | 11 | Emmit metrics point to a tranquility node which indexes it into the cluster back again. 12 | 13 | #Run it (currently only tested on minikube) 14 | - checkout repo 15 | - build docker images, see `build_images.sh` 16 | - `helm init` (assumes working kubectl, for example with `minikube start`) 17 | - `helm dep update` Pull in deps 18 | - `helm upgrade --debug development druid/ --install` Start Cluster, takes a while till zookeeper is fully functional (~5min) 19 | - Access the pivot UI `minikube service druid-pivot-service` and you should see the Metric Datasource which are druids internal metrics. 20 | - Delete the Cluster with `helm delete development --purge ` PersistentStorage from the zK must be deleted manually. 21 | 22 | 23 | 24 | #Assorted Notes: 25 | 26 | #Install Mini cube 27 | curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.12.2/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ 28 | #Install kubectl 29 | curl -Lo kubectl http://storage.googleapis.com/kubernetes-release/release/v1.3.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ 30 | 31 | #Completion 32 | brew install bash-completion 33 | 34 | 35 | #Install Helm-client 36 | brew cask install helm 37 | 38 | 39 | 40 | minikube start 41 | eval $(minikube docker-env) 42 | 43 | #Deployments, not used anymore 44 | kubectl create -f local-zk/Deployment.yml 45 | 46 | kubectl create -f metadata-mysql/Deployment.yml 47 | 48 | kubectl create -f coordinator/Deployment.yml 49 | 50 | kubectl create -f overlord/Deployment.yml 51 | 52 | kubectl create -f middleManager/Deployment.yml 53 | 54 | kubectl create -f historical/Deployment.yml 55 | 56 | kubectl create -f broker/Deployment.yml 57 | 58 | kubectl create -f pivot/Deployment.yml 59 | 60 | 61 | minikube service druid-quickstart-service 62 | 63 | 64 | #Jenkins 65 | helm install stable/jenkins 66 | minikube service hoping-kitten-jenkins --url 67 | printf $(printf '\%o' `kubectl get secret --namespace default hoping-kitten-jenkins -o jsonpath="{.data.jenkins-admin-password[*]}"`);echo 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /druid/templates/historical.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: "{{ .Chart.Name }}-{{ .Values.nodes.historical.name }}-deployment" 5 | labels: 6 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 7 | spec: 8 | replicas: {{ .Values.nodes.historical.replicaCount }} 9 | template: 10 | metadata: 11 | labels: 12 | name: "{{ .Chart.Name }}-{{ .Values.nodes.historical.name }}-rc" 13 | app: {{ template "fullname" . }} 14 | spec: 15 | containers: 16 | - name: "{{ .Chart.Name }}-{{ .Values.nodes.historical.name }}" 17 | image: "{{ .Values.nodes.historical.image.repository }}:{{ .Values.nodes.historical.image.tag }}" 18 | imagePullPolicy: {{ .Values.nodes.historical.image.pullPolicy }} 19 | ports: 20 | {{range .Values.nodes.historical.service.ports }} 21 | - containerPort: {{ .port }} 22 | {{end}} 23 | #TODO add readiness & liveness probes 24 | volumeMounts: 25 | - name: data 26 | mountPath: /var/druid/segments 27 | - name: segment-cache 28 | mountPath: /var/druid/segment-cache 29 | - name: config-volume 30 | mountPath: "/iap-druid/{{ .Values.iapVersion }}/conf/druid/" 31 | volumes: 32 | - name: data 33 | persistentVolumeClaim: 34 | claimName: {{ .Values.druidSegmentDeepStorage }} 35 | - name: segment-cache 36 | persistentVolumeClaim: 37 | claimName: {{ .Values.druidSegmentCache }} 38 | - name: config-volume 39 | configMap: 40 | name: common-runtime-configmap 41 | items: 42 | - key: common.runtime.properties 43 | path: _common/common.runtime.properties 44 | - key: "{{ .Values.nodes.historical.name }}.runtime.properties" 45 | path: historical/runtime.properties 46 | - key: "{{ .Values.nodes.historical.name }}.jvm.config" 47 | path: historical/jvm.config 48 | - key: "{{ .Values.nodes.historical.name }}.main.config" 49 | path: historical/main.config 50 | --- 51 | apiVersion: v1 52 | kind: Service 53 | metadata: 54 | name: {{ .Values.nodes.historical.service.name }} 55 | #TODO flexible service Name, configMap 56 | labels: 57 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 58 | spec: 59 | type: {{ .Values.nodes.historical.service.type }} 60 | ports: 61 | {{range .Values.nodes.historical.service.ports }} 62 | - name: {{ .name }} 63 | port: {{ .port }} 64 | {{end}} 65 | selector: 66 | name: "{{ .Chart.Name }}-{{ .Values.nodes.historical.name }}-rc" -------------------------------------------------------------------------------- /dockerfiles/base/common.runtime.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Extensions 3 | # 4 | 5 | druid.extensions.directory=dist/druid/extensions 6 | druid.extensions.hadoopDependenciesDir=dist/druid/hadoop-dependencies 7 | druid.extensions.loadList=["mysql-metadata-storage"] 8 | 9 | # 10 | # Logging 11 | # 12 | 13 | # Log all runtime properties on startup. Disable to avoid logging properties on startup: 14 | druid.startup.logging.logProperties=true 15 | 16 | # 17 | # Zookeeper 18 | # 19 | 20 | druid.zk.service.host=druid-local-zk-service 21 | druid.zk.paths.base=/druid 22 | 23 | # 24 | # Metadata storage 25 | # 26 | 27 | # For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): 28 | #druid.metadata.storage.type=derby 29 | #druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true 30 | #druid.metadata.storage.connector.host=localhost 31 | #druid.metadata.storage.connector.port=1527 32 | 33 | # For MySQL: 34 | druid.metadata.storage.type=mysql 35 | druid.metadata.storage.connector.connectURI=jdbc:mysql://druid-metadata-mysql-service:3306/druid 36 | druid.metadata.storage.connector.user=druid 37 | druid.metadata.storage.connector.password=secret 38 | druid.metadata.storage.connector.createTables=true 39 | 40 | # For PostgreSQL: 41 | #druid.metadata.storage.type=postgresql 42 | #druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid 43 | #druid.metadata.storage.connector.user=... 44 | #druid.metadata.storage.connector.password=... 45 | 46 | # 47 | # Deep storage 48 | # 49 | 50 | # For local disk (only viable in a cluster if this is a network mount): 51 | druid.storage.type=local 52 | druid.storage.storageDirectory=var/druid/segments 53 | 54 | # For HDFS: 55 | #druid.storage.type=hdfs 56 | #druid.storage.storageDirectory=/druid/segments 57 | 58 | # For S3: 59 | #druid.storage.type=s3 60 | #druid.storage.bucket=your-bucket 61 | #druid.storage.baseKey=druid/segments 62 | #druid.s3.accessKey=... 63 | #druid.s3.secretKey=... 64 | 65 | # 66 | # Indexing service logs 67 | # 68 | 69 | # For local disk (only viable in a cluster if this is a network mount): 70 | druid.indexer.logs.type=file 71 | druid.indexer.logs.directory=var/druid/indexing-logs 72 | 73 | # For HDFS: 74 | #druid.indexer.logs.type=hdfs 75 | #druid.indexer.logs.directory=/druid/indexing-logs 76 | 77 | # For S3: 78 | #druid.indexer.logs.type=s3 79 | #druid.indexer.logs.s3Bucket=your-bucket 80 | #druid.indexer.logs.s3Prefix=druid/indexing-logs 81 | 82 | # 83 | # Service discovery 84 | # 85 | 86 | druid.selectors.indexing.serviceName=druid/overlord 87 | druid.selectors.coordinator.serviceName=druid/coordinator 88 | 89 | # 90 | # Monitoring 91 | # 92 | 93 | druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"] 94 | druid.emitter=logging 95 | druid.emitter.logging.logLevel=info 96 | -------------------------------------------------------------------------------- /druid/templates/config/_common-runtime.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Extensions 3 | # 4 | 5 | druid.extensions.directory=dist/druid/extensions 6 | druid.extensions.hadoopDependenciesDir=dist/druid/hadoop-dependencies 7 | druid.extensions.loadList=["postgresql-metadata-storage"] 8 | 9 | 10 | # 11 | # Logging 12 | # 13 | 14 | # Log all runtime properties on startup. Disable to avoid logging properties on startup: 15 | druid.startup.logging.logProperties=true 16 | 17 | # 18 | # Zookeeper 19 | # 20 | 21 | druid.zk.service.host={{ printf "%s-%s" .Release.Name "zk" | trunc 24 }} 22 | druid.zk.paths.base=/druid 23 | 24 | # 25 | # Metadata storage 26 | # 27 | 28 | # For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): 29 | #druid.metadata.storage.type=derby 30 | #druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true 31 | #druid.metadata.storage.connector.host=localhost 32 | #druid.metadata.storage.connector.port=1527 33 | 34 | 35 | #TODO Variable Handling of User/PW 36 | # For MySQL: 37 | #druid.metadata.storage.type=mysql 38 | #druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid 39 | #druid.metadata.storage.connector.user=druid 40 | #druid.metadata.storage.connector.password=secret 41 | #druid.metadata.storage.connector.createTables=true 42 | 43 | # For PostgreSQL: 44 | druid.metadata.storage.type=postgresql 45 | druid.metadata.storage.connector.connectURI=jdbc:postgresql://{{ printf "%s-%s" .Release.Name "postgresql" | trunc 24 }}:5432/druid 46 | druid.metadata.storage.connector.user=druid 47 | druid.metadata.storage.connector.password=secret 48 | druid.metadata.storage.connector.createTables=true 49 | # 50 | # Deep storage 51 | # 52 | 53 | # For local disk (only viable in a cluster if this is a network mount): 54 | druid.storage.type=local 55 | druid.storage.storageDirectory=var/druid/segments 56 | 57 | # For HDFS: 58 | #druid.storage.type=hdfs 59 | #druid.storage.storageDirectory=/druid/segments 60 | 61 | # For S3: 62 | #druid.storage.type=s3 63 | #druid.storage.bucket=your-bucket 64 | #druid.storage.baseKey=druid/segments 65 | #druid.s3.accessKey=... 66 | #druid.s3.secretKey=... 67 | 68 | # 69 | # Indexing service logs 70 | # 71 | 72 | # For local disk (only viable in a cluster if this is a network mount): 73 | druid.indexer.logs.type=file 74 | druid.indexer.logs.directory=var/druid/indexing-logs 75 | 76 | # For HDFS: 77 | #druid.indexer.logs.type=hdfs 78 | #druid.indexer.logs.directory=/druid/indexing-logs 79 | 80 | # For S3: 81 | #druid.indexer.logs.type=s3 82 | #druid.indexer.logs.s3Bucket=your-bucket 83 | #druid.indexer.logs.s3Prefix=druid/indexing-logs 84 | 85 | # 86 | # Service discovery 87 | # 88 | 89 | druid.selectors.indexing.serviceName=druid/overlord 90 | druid.selectors.coordinator.serviceName=druid/coordinator 91 | 92 | # 93 | # Monitoring 94 | # 95 | 96 | druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"] 97 | druid.emitter=http 98 | druid.emitter.logging.logLevel=info 99 | druid.emitter.http.recipientBaseUrl=http://{{ .Values.nodes.tranquility.service.name }}:8200/v1/post/metrics -------------------------------------------------------------------------------- /druid/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for druid. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | replicaCount: 1 5 | 6 | #TODO remove quickstart stuff 7 | iapVersion: "imply-1.3.1" 8 | debug: "True" 9 | 10 | postgresqlName: "postgresql" 11 | 12 | druidSegmentDeepStorage: "druid-deep-storage" 13 | druidSegmentCache: "druid-deep-segment-cache" 14 | 15 | 16 | zookeeper: 17 | Replicas: 3 18 | Name: zk 19 | postgresql: 20 | postgresUser: druid 21 | postgresDatabase: druid 22 | postgresPassword: secret 23 | 24 | nodes: 25 | coordinator: 26 | name: "coordinator" 27 | replicaCount: 1 28 | image: 29 | repository: jkaralus/druid8s-coordinator 30 | tag: latest 31 | pullPolicy: IfNotPresent 32 | service: 33 | name: druid-coordinator-service 34 | type: NodePort 35 | ports: 36 | - name: coordinator-port 37 | port: 8081 38 | overlord: 39 | name: "overlord" 40 | replicaCount: 1 41 | image: 42 | repository: jkaralus/druid8s-overlord 43 | tag: latest 44 | pullPolicy: IfNotPresent 45 | service: 46 | name: druid-overlord-service 47 | type: NodePort 48 | ports: 49 | - name: overlord-port 50 | port: 8090 51 | historical: 52 | name: "historical" 53 | replicaCount: 1 54 | image: 55 | repository: jkaralus/druid8s-historical 56 | tag: latest 57 | pullPolicy: IfNotPresent 58 | service: 59 | name: druid-historical-service 60 | type: ClusterIP 61 | ports: 62 | - name: historical-port 63 | port: 8083 64 | broker: 65 | name: "broker" 66 | replicaCount: 1 67 | image: 68 | repository: jkaralus/druid8s-broker 69 | tag: latest 70 | pullPolicy: IfNotPresent 71 | service: 72 | name: druid-broker-service 73 | type: NodePort 74 | ports: 75 | - name: broker-port 76 | port: 8082 77 | tranquility: 78 | name: "tranquility" 79 | replicaCount: 1 80 | image: 81 | repository: jkaralus/druid8s-tranquility 82 | tag: latest 83 | pullPolicy: IfNotPresent 84 | service: 85 | name: druid-tranquility-service 86 | type: NodePort 87 | ports: 88 | - name: tranquility-port 89 | port: 8200 90 | pivot: 91 | name: "pivot" 92 | replicaCount: 1 93 | image: 94 | repository: jkaralus/druid8s-pivot 95 | tag: latest 96 | pullPolicy: IfNotPresent 97 | service: 98 | name: druid-pivot-service 99 | type: NodePort 100 | ports: 101 | - name: pivot-port 102 | port: 9095 103 | middlemanager: 104 | name: "middlemanager" 105 | replicaCount: 1 106 | image: 107 | repository: jkaralus/druid8s-middlemanager 108 | tag: latest 109 | pullPolicy: IfNotPresent 110 | service: 111 | name: druid-middlemanager-service 112 | type: NodePort 113 | ports: 114 | - name: middlemanager-port 115 | port: 8091 116 | - name: peon-0 117 | port: 8100 118 | - name: peon-1 119 | port: 8101 120 | - name: peon-2 121 | port: 8102 122 | - name: peon-3 123 | port: 8103 124 | 125 | image: 126 | historical: 127 | repository: jkaralus/druid8s-historical 128 | tag: latest 129 | pullPolicy: IfNotPresent 130 | quickstart: 131 | repository: jkaralus/druid8s-quickstart 132 | tag: latest 133 | pullPolicy: IfNotPresent 134 | service: 135 | historical: 136 | name: historical 137 | type: ClusterIP 138 | externalPort: 8083 139 | internalPort: 8083 140 | quickstart: 141 | name: quickstart 142 | type: NodePort 143 | port: 9095 144 | #TODO remove ulgy hotfix 145 | type: NodePort 146 | 147 | 148 | 149 | 150 | 151 | resources: 152 | limits: 153 | cpu: 100m 154 | memory: 512Mi 155 | requests: 156 | cpu: 100m 157 | memory: 256Mi 158 | 159 | --------------------------------------------------------------------------------