├── mongo ├── .env ├── db-init │ ├── 01-create-collection.js │ └── 00-create-user.js ├── secret.sh ├── restheart │ └── users.yml ├── secret.key └── docker-compose.yml ├── mysql5 ├── .env ├── init.d │ └── main.sql ├── cnf │ └── local.cnf ├── docker-compose.yml ├── mysql.cnf └── sql │ └── queries.sql ├── oracle ├── .env ├── docker-compose.yml ├── init.d │ └── setup │ │ └── logminer.sql └── README.md ├── kafka-lb ├── .env ├── nginx │ ├── Dockerfile │ ├── broker_latency.sh │ ├── broker-1.conf │ ├── broker-2.conf │ ├── broker-3.conf │ └── broker-4.conf └── docker-compose.yml ├── storage ├── .env ├── README.md ├── mc-config.json └── docker-compose.yml ├── bin ├── config │ └── ksql.properties ├── confluent-hub ├── druid_query.sh ├── ksql-shell ├── connect-all ├── down.sh ├── connect-log ├── pinot ├── druid ├── ksql-restart └── connect ├── dashboards ├── .env ├── turnilo │ ├── Dockerfile │ └── entrypoint.sh ├── superset │ ├── superset_config.py │ └── Dockerfile └── docker-compose.yml ├── kafka ├── jmx_prometheus │ ├── jolokia-jvm.jar │ ├── jmx_prometheus_javaagent.jar │ ├── jolokia-jvm-1.7.2.jar │ ├── jmx_prometheus_javaagent-0.20.0.jar │ ├── kafka-config.yml │ └── sr-config.yml ├── .env └── docker-compose.yml ├── kafka-raft ├── jmx_prometheus │ ├── jolokia-jvm.jar │ ├── jmx_prometheus_javaagent.jar │ ├── jolokia-jvm-1.7.2.jar │ ├── jmx_prometheus_javaagent-0.20.0.jar │ ├── kafka-config.yml │ └── sr-config.yml ├── .env └── docker-compose.yml ├── kafka-zk ├── jmx_prometheus │ ├── jolokia-jvm.jar │ ├── jmx_prometheus_javaagent.jar │ ├── jolokia-jvm-1.7.2.jar │ ├── jmx_prometheus_javaagent-0.20.0.jar │ ├── kafka-config.yml │ └── sr-config.yml ├── .env └── docker-compose.yml ├── postgres ├── .env └── docker-compose.yml ├── connect ├── .env ├── secrets │ ├── mysql.properties │ └── postgres.properties ├── jmx_prometheus │ ├── jmx_prometheus_javaagent.jar │ └── connect-config.yml └── docker-compose.yml ├── ksqlDB ├── .env ├── jmx_prometheus │ ├── jmx_prometheus_javaagent.jar │ └── streams-config.yml └── docker-compose.yml ├── monitoring ├── .env ├── static │ ├── consumer_request_metrics.png │ └── producer_request_metrics.png ├── kowl │ └── config.yaml ├── grafana │ ├── prometheus.yml │ ├── dashboards.yml │ ├── druid.yml │ └── dashboards │ │ ├── home.json │ │ ├── druid │ │ └── druid-ingestion.json │ │ ├── streams │ │ └── streams_process_blank.json │ │ └── kafka │ │ └── 11_kafka_cluster_request_2.json ├── akhq │ └── application.yml ├── prometheus │ ├── targets.json │ └── prometheus.yml ├── kouncil │ └── kouncil.yaml ├── cmak │ └── Dockerfile └── docker-compose.yml ├── elasticsearch ├── .env └── docker-compose.yml ├── sqlserver ├── README.md ├── .env └── docker-compose.yml ├── flink ├── jmx_prometheus │ ├── flink-config.yml │ └── jmx_prometheus_javaagent.jar ├── .env └── docker-compose.yml ├── druid ├── jmx_prometheus │ ├── druid-config.yml │ └── jmx_prometheus_javaagent.jar ├── .env ├── scripts │ ├── druid_download_extensions.sh │ └── druid_dump.sh ├── environment └── docker-compose.yml ├── pinot ├── .env ├── configs │ ├── pinot-controller.conf │ └── pinot-server.conf └── docker-compose.yml ├── mysql ├── cnf │ └── local.cnf ├── init.d │ ├── backup.sql │ └── main.sql ├── .env └── docker-compose.yml ├── scripts ├── brokers.sh ├── ips.sh ├── install_connect-hub.sh ├── install_jmxterm.sh ├── volume_sizes.sh ├── .zshrc ├── stats.sh └── container.sh ├── .gitignore ├── network.sh ├── opensearch └── docker-compose.yml ├── cassandra └── docker-compose.yml └── README.md /mongo/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-mongo 2 | -------------------------------------------------------------------------------- /mysql5/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-mysql5 2 | -------------------------------------------------------------------------------- /oracle/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-oracle 2 | -------------------------------------------------------------------------------- /kafka-lb/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-kafka_lb 2 | -------------------------------------------------------------------------------- /storage/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-buckets 2 | -------------------------------------------------------------------------------- /bin/config/ksql.properties: -------------------------------------------------------------------------------- 1 | auto.offset.reset=earliest 2 | -------------------------------------------------------------------------------- /dashboards/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-dashboards 2 | -------------------------------------------------------------------------------- /kafka/jmx_prometheus/jolokia-jvm.jar: -------------------------------------------------------------------------------- 1 | jolokia-jvm-1.7.2.jar -------------------------------------------------------------------------------- /kafka-raft/jmx_prometheus/jolokia-jvm.jar: -------------------------------------------------------------------------------- 1 | jolokia-jvm-1.7.2.jar -------------------------------------------------------------------------------- /kafka-zk/jmx_prometheus/jolokia-jvm.jar: -------------------------------------------------------------------------------- 1 | jolokia-jvm-1.7.2.jar -------------------------------------------------------------------------------- /mongo/db-init/01-create-collection.js: -------------------------------------------------------------------------------- 1 | db.createCollection("orders") 2 | -------------------------------------------------------------------------------- /postgres/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-postgres 2 | 3 | VERSION=14.6 4 | -------------------------------------------------------------------------------- /connect/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-connect 2 | 3 | VERSION=7.7.0 4 | 5 | -------------------------------------------------------------------------------- /ksqlDB/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-ksqldb 2 | 3 | VERSION=7.6.0 4 | 5 | -------------------------------------------------------------------------------- /monitoring/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-monitoring 2 | 3 | VERSION=9.3.2 4 | -------------------------------------------------------------------------------- /elasticsearch/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-elasticsearch 2 | 3 | VERSION=8.5.3 4 | -------------------------------------------------------------------------------- /kafka-zk/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- 1 | jmx_prometheus_javaagent-0.20.0.jar -------------------------------------------------------------------------------- /kafka/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- 1 | jmx_prometheus_javaagent-0.20.0.jar -------------------------------------------------------------------------------- /kafka-raft/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- 1 | jmx_prometheus_javaagent-0.20.0.jar -------------------------------------------------------------------------------- /sqlserver/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Use Rosetta for x86/amd64 emulation on Apple Silicon 4 | 5 | -------------------------------------------------------------------------------- /storage/README.md: -------------------------------------------------------------------------------- 1 | aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket foo 2 | -------------------------------------------------------------------------------- /flink/jmx_prometheus/flink-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | rules: 4 | - pattern: .* 5 | -------------------------------------------------------------------------------- /mysql5/init.d/main.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE main; 2 | GRANT ALL PRIVILEGES ON main.* TO 'user'@'%'; 3 | USE main; 4 | -------------------------------------------------------------------------------- /druid/jmx_prometheus/druid-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | rules: 4 | - pattern: java.lang 5 | -------------------------------------------------------------------------------- /flink/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-flink 2 | 3 | #VERSION="1.16.1-scala_2.12-java11" 4 | VERSION="1.18.1-scala_2.12-java17" 5 | -------------------------------------------------------------------------------- /kafka-zk/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-kafka 2 | 3 | VERSION=7.7.0 4 | 5 | #PLATFORM=linux/arm64/v8 6 | #PLATFORM=linux/x86_64 7 | -------------------------------------------------------------------------------- /kafka/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-kafka 2 | 3 | VERSION=7.7.0 4 | 5 | #PLATFORM=linux/arm64/v8 6 | #PLATFORM=linux/x86_64 7 | -------------------------------------------------------------------------------- /pinot/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-pinot 2 | 3 | #VERSION=0.11.0 4 | VERSION=1.1.0-21-openjdk 5 | 6 | #PLATFORM=linux/arm64 7 | -------------------------------------------------------------------------------- /kafka-raft/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-kafka 2 | 3 | VERSION=7.7.0 4 | 5 | #PLATFORM=linux/arm64/v8 6 | #PLATFORM=linux/x86_64 7 | -------------------------------------------------------------------------------- /mysql/cnf/local.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | local-infile 3 | secure-file-priv=/data 4 | #secure-file-priv = "" 5 | 6 | [mysql] 7 | local-infile 8 | -------------------------------------------------------------------------------- /mysql5/cnf/local.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | local-infile 3 | secure-file-priv=/data 4 | #secure-file-priv = "" 5 | 6 | [mysql] 7 | local-infile 8 | -------------------------------------------------------------------------------- /connect/secrets/mysql.properties: -------------------------------------------------------------------------------- 1 | CONNECTION_URL=jdbc:mysql://mysql:3306/MAIN 2 | CONNECTION_USER=user 3 | CONNECTION_PASSWORD=userpw 4 | 5 | -------------------------------------------------------------------------------- /kafka/jmx_prometheus/jolokia-jvm-1.7.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/kafka/jmx_prometheus/jolokia-jvm-1.7.2.jar -------------------------------------------------------------------------------- /mysql/init.d/backup.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE DATABASE BACKUP; 3 | 4 | GRANT ALL PRIVILEGES ON BACKUP.* TO 'user'@'%'; 5 | 6 | USE BACKUP; 7 | 8 | -------------------------------------------------------------------------------- /kafka-zk/jmx_prometheus/jolokia-jvm-1.7.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/kafka-zk/jmx_prometheus/jolokia-jvm-1.7.2.jar -------------------------------------------------------------------------------- /mongo/db-init/00-create-user.js: -------------------------------------------------------------------------------- 1 | db.createUser({ 2 | user: "admin", 3 | pwd: "mongo", 4 | roles: [ { role: "readWrite", db: "admin" } ] 5 | }) 6 | -------------------------------------------------------------------------------- /connect/secrets/postgres.properties: -------------------------------------------------------------------------------- 1 | CONNECTION_URL=jdbc:postgresql://postgres:5432/main 2 | CONNECTION_USER=admin 3 | CONNECTION_PASSWORD=postgres 4 | -------------------------------------------------------------------------------- /druid/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-druid 2 | 3 | #VERSION=0.23.0 4 | #VERSION=25.0.0-arm64 5 | 6 | #VERSION=30.0.1 7 | VERSION=30.0.1-arm64 8 | 9 | -------------------------------------------------------------------------------- /kafka-raft/jmx_prometheus/jolokia-jvm-1.7.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/kafka-raft/jmx_prometheus/jolokia-jvm-1.7.2.jar -------------------------------------------------------------------------------- /monitoring/static/consumer_request_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/monitoring/static/consumer_request_metrics.png -------------------------------------------------------------------------------- /monitoring/static/producer_request_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/monitoring/static/producer_request_metrics.png -------------------------------------------------------------------------------- /druid/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/druid/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /flink/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/flink/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /ksqlDB/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/ksqlDB/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /connect/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/connect/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /mongo/secret.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # means to generate a new secret key 5 | # 6 | 7 | openssl rand -base64 756 > secret.key 8 | chmod 600 ./secret.key 9 | -------------------------------------------------------------------------------- /kafka/jmx_prometheus/jmx_prometheus_javaagent-0.20.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/kafka/jmx_prometheus/jmx_prometheus_javaagent-0.20.0.jar -------------------------------------------------------------------------------- /dashboards/turnilo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:alpine3.14 2 | 3 | RUN npm install -g turnilo 4 | 5 | COPY ./entrypoint.sh /entrypoint.sh 6 | 7 | ENTRYPOINT /entrypoint.sh 8 | 9 | -------------------------------------------------------------------------------- /kafka-zk/jmx_prometheus/jmx_prometheus_javaagent-0.20.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/kafka-zk/jmx_prometheus/jmx_prometheus_javaagent-0.20.0.jar -------------------------------------------------------------------------------- /kafka-raft/jmx_prometheus/jmx_prometheus_javaagent-0.20.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kineticedge/dev-local/HEAD/kafka-raft/jmx_prometheus/jmx_prometheus_javaagent-0.20.0.jar -------------------------------------------------------------------------------- /mysql/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-mysql 2 | 3 | PLATFORM=linux/arm64/v8 4 | IMAGE=arm64v8/mysql:8.0.29-oracle 5 | 6 | #PLATFORM=linux/x86_64 7 | #IMAGE=mysql:8.0.29 8 | 9 | 10 | -------------------------------------------------------------------------------- /monitoring/kowl/config.yaml: -------------------------------------------------------------------------------- 1 | kafka: 2 | brokers: 3 | - broker-1:9092 4 | - broker-2:9092 5 | - broker-3:9092 6 | - broker-4:9092 7 | 8 | # server: 9 | # listenPort: 8080 10 | -------------------------------------------------------------------------------- /mysql/init.d/main.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE DATABASE main; 3 | 4 | GRANT ALL PRIVILEGES ON main.* TO 'user'@'%' with grant option; 5 | 6 | GRANT SUPER ON *.* TO 'user'@'%'; 7 | 8 | USE main; 9 | 10 | -------------------------------------------------------------------------------- /sqlserver/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dev-sqlserver 2 | 3 | VERSION=2022-latest 4 | #VERSION=2022-preview-ubuntu-22.04 5 | 6 | # not a suprrise that Microsoft does not have an arm64 image. 7 | PLATFORM=linux/amd64 8 | -------------------------------------------------------------------------------- /monitoring/grafana/prometheus.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: prometheus 5 | type: prometheus 6 | url: http://prometheus:9090 7 | access: proxy 8 | editable: false 9 | 10 | -------------------------------------------------------------------------------- /monitoring/akhq/application.yml: -------------------------------------------------------------------------------- 1 | akhq: 2 | connections: 3 | kafka: 4 | properties: 5 | bootstrap.servers: "broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092" 6 | security.protocol: PLAINTEXT 7 | 8 | -------------------------------------------------------------------------------- /dashboards/turnilo/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "${DRUID}" == "" ]; then 4 | echo "environment variable DRUID is not defined." 5 | exit 6 | fi 7 | 8 | echo "Druid URL ${DRUID}" 9 | 10 | turnilo --druid ${DRUID} 11 | 12 | -------------------------------------------------------------------------------- /kafka-lb/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:latest 2 | 3 | RUN \ 4 | apt-get update && \ 5 | apt-get install -y curl iproute2 iputils-ping 6 | 7 | #RUN \ 8 | # apt-get install -y blockade 9 | 10 | 11 | COPY ./broker_latency.sh /root/ 12 | -------------------------------------------------------------------------------- /bin/confluent-hub: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | "$(dirname -- "$0")" 4 | 5 | mkdir -p ../tmp 6 | touch ../tmp/connect-distributed.properties 7 | 8 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir . --no-prompt "${@}" 9 | -------------------------------------------------------------------------------- /scripts/brokers.sh: -------------------------------------------------------------------------------- 1 | 2 | IMAGE="confluentinc/cp-kafkacat:latest" 3 | 4 | docker pull "${IMAGE}" --platform "linux/amd64/v8" 5 | 6 | docker run --tty --network dev-local --platform "linux/amd64" "${IMAGE}" \ 7 | kafkacat -b broker-1:9092 -L 8 | 9 | -------------------------------------------------------------------------------- /storage/mc-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "10", 3 | "aliases": { 4 | "minio": { 5 | "url": "http://minio:9000", 6 | "accessKey": "admin", 7 | "secretKey": "miniominio", 8 | "api": "s3v4", 9 | "path": "auto" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /bin/druid_query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DATA=$1 4 | shift 5 | 6 | URL=http://localhost:48082 7 | #URL=http://localhost:48081 8 | 9 | CONTENT_TYPE_HDR='Content-Type:application/json' 10 | 11 | curl -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/v2/sql/ 12 | 13 | -------------------------------------------------------------------------------- /mongo/restheart/users.yml: -------------------------------------------------------------------------------- 1 | ## configuration file for simpleFileAuthenticator 2 | --- 3 | users: 4 | - userid: user 5 | password: restheart 6 | roles: [user] 7 | 8 | - userid: admin 9 | password: restheart 10 | roles: [admin] 11 | 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | .gradle 4 | grafana.db 5 | *.bak 6 | build 7 | jmxterm.jar 8 | /connect/secrets/*.properties 9 | 10 | /tmp 11 | /*/data/* 12 | /ksqlDB/ext/* 13 | /connect/jars/* 14 | /connect/tmp/* 15 | /druid/storage/* 16 | /druid/staging/* 17 | /flink/jars/* 18 | /flink/tmp/* 19 | 20 | -------------------------------------------------------------------------------- /monitoring/grafana/dashboards.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'default' 5 | orgId: 1 6 | folder: '' 7 | folderUid: '' 8 | type: file 9 | disableDeletion: true 10 | editable: false 11 | allowUiUpdates: false 12 | updateIntervalSeconds: 5 13 | options: 14 | path: /var/lib/grafana/dashboards 15 | -------------------------------------------------------------------------------- /network.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NETWORK=dev-local 4 | 5 | # 6 | # creates a network unique to this project that can be shared between docker compose instances 7 | # 8 | if [ "$(docker network inspect -f '{{.Name}}' $NETWORK 2>/dev/null)" != "$NETWORK" ]; then 9 | echo "creating network $NETWORK" 10 | (docker network create $NETWORK >/dev/null) 11 | fi 12 | 13 | -------------------------------------------------------------------------------- /scripts/ips.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | containers=$(docker ps --format '{{.Names}}|{{.ID}}' | sort) 4 | 5 | for container in $containers; do 6 | 7 | id=$(echo $container | cut -d\| -f2) 8 | name=$(echo $container | cut -d\| -f1) 9 | 10 | ip=$(docker inspect -f '{{.Name}} {{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $id) 11 | 12 | echo $ip 13 | 14 | done 15 | 16 | 17 | -------------------------------------------------------------------------------- /monitoring/prometheus/targets.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "targets": [ 4 | ], 5 | "labels": { 6 | "job": "producers" 7 | } 8 | }, 9 | { 10 | "targets": [ 11 | ], 12 | "labels": { 13 | "job": "consumers" 14 | } 15 | }, 16 | { 17 | "targets": [ 18 | "stream:7071" 19 | ], 20 | "labels": { 21 | "job": "streams" 22 | } 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /scripts/install_connect-hub.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname $0) 4 | 5 | CWD=$(pwd) 6 | 7 | mkdir -p ./tmp 8 | 9 | if [ ! -f ./tmp/confluent-hub-client.tar.gz ]; then 10 | echo "downloading..." 11 | curl -s -o ./tmp/confluent-hub-client.tar.gz http://client.hub.confluent.io/confluent-hub-client-latest.tar.gz 12 | fi 13 | 14 | (cd /usr/local/confluent; gzcat ${CWD}/tmp/confluent-hub-client.tar.gz | tar xfv -) 15 | -------------------------------------------------------------------------------- /monitoring/kouncil/kouncil.yaml: -------------------------------------------------------------------------------- 1 | kouncil: 2 | clusters: 3 | - name: kafka 4 | brokers: 5 | - host: broker-1 6 | port: 9092 7 | jmxPort: 7071 8 | - host: broker-2 9 | port: 9092 10 | jmxPort: 7071 11 | - host: broker-3 12 | port: 9092 13 | jmxPort: 7071 14 | - host: broker-4 15 | port: 9092 16 | jmxPort: 7071 17 | -------------------------------------------------------------------------------- /bin/ksql-shell: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 ksql-file" 6 | echo "" 7 | } 8 | 9 | if [ $# -lt 1 ]; then 10 | usage 11 | exit 12 | fi 13 | 14 | DIR=$(dirname $0) 15 | 16 | FILE=$1 17 | shift 18 | 19 | if [ ! -f $FILE ]; then 20 | echo "" 21 | echo "unable to find ksql file." 22 | echo "" 23 | exit 24 | fi 25 | 26 | ksql --config-file $DIR/config/ksql.properties --file=$FILE 27 | 28 | 29 | -------------------------------------------------------------------------------- /monitoring/grafana/druid.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: druid 5 | type: grafadruid-druid-datasource 6 | # this plugin actually does not use 'url', but by adding it it shows up nicely in the list of datasources 7 | url: http://druid-router:8888 8 | jsonData: 9 | # this is the actual propety used by this plugin for communicating with the druid router 10 | connection.url: "http://druid-router:8888" 11 | editable: false 12 | -------------------------------------------------------------------------------- /dashboards/superset/superset_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | WTF_CSRF_ENABLED = False 5 | 6 | # 7 | # Disable Authorization 8 | # 9 | 10 | #AUTH_ROLE_PUBLIC = 'Public' 11 | #AUTH_USER_REGISTRATION_ROLE = 'Public' 12 | #PUBLIC_ROLE_LIKE_GAMMA = True 13 | 14 | #PUBLIC_ROLE_LIKE 15 | 16 | #Allow for Embedded Javascript within charts that allow it 17 | ENABLE_JAVASCRIPT_CONTROLS = True 18 | 19 | #To Avoid hard-cording of personal MAPBOX_API_KEY into the local image, reference from evironment variable. 20 | MAPBOX_API_KEY = os.getenv('MAPBOX_API_KEY', '') 21 | -------------------------------------------------------------------------------- /bin/connect-all: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ $# -lt 2 ]; then 4 | echo "" 5 | echo "Usage: " 6 | echo " $0 {create|pause|resume|delete|status|validate} directory" 7 | echo "" 8 | exit 9 | fi 10 | 11 | BIN_DIR=$(dirname $0) 12 | BASE_DIR=$(dirname $BIN_DIR) 13 | 14 | if [ $# -lt 1 ]; then 15 | echo "usage: $0 {create|pause|resume|delete|status|validate} " 16 | exit 17 | fi 18 | 19 | command=$1 20 | shift 21 | 22 | directory=$1 23 | shift 24 | 25 | declare -a connectors=($(ls $directory/*.json)) 26 | 27 | for i in "${connectors[@]}"; do 28 | ${BIN_DIR}/connect $command $i 29 | done 30 | 31 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker_latency.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LATENCY="${1}ms" 4 | shift 5 | 6 | echo "latency: $LATENCY" 7 | 8 | tc qdisc del dev eth0 root 9 | tc qdisc add dev eth0 root handle 1: prio 10 | tc qdisc add dev eth0 parent 1:3 handle 30: netem delay ${LATENCY} 11 | 12 | 13 | declare -a HOSTS=(broker-1 broker-2 broker-3 broker-4) 14 | 15 | for host in "${HOSTS[@]}"; do 16 | 17 | ip=$(ping -c 1 $host | head -1 | awk -F'[()]' '/PING/{print $2}') 18 | 19 | echo "$host : $ip" 20 | tc filter add dev eth0 protocol ip parent 1:0 prio 3 u32 match ip dst $ip flowid 1:3 21 | 22 | done 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /monitoring/cmak/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11 2 | 3 | ENV ZK_HOSTS=localhost:2181 4 | ENV CMAK_VERSION=3.0.0.5 5 | 6 | RUN cd /tmp && wget https://github.com/yahoo/CMAK/archive/${CMAK_VERSION}.tar.gz && \ 7 | tar -xzvf ${CMAK_VERSION}.tar.gz 8 | 9 | RUN cd /tmp/CMAK-${CMAK_VERSION} && \ 10 | ./sbt clean dist 11 | 12 | RUN unzip -d / /tmp/CMAK-${CMAK_VERSION}/target/universal/cmak-${CMAK_VERSION}.zip 13 | 14 | RUN rm -fr /tmp/CMAK-${CMAK_VERSION} /tmp/${CMAK_VERSION}.tar.gz 15 | 16 | WORKDIR /cmak-${CMAK_VERSION} 17 | 18 | EXPOSE 9001 19 | ENTRYPOINT ["./bin/cmak","-Dconfig.file=conf/application.conf"] 20 | -------------------------------------------------------------------------------- /oracle/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | data: {} 9 | fra: {} 10 | 11 | services: 12 | 13 | oracle: 14 | image: oracle/database:19.3.0-se2 15 | #image: oracle/database:12.2.0.1-se2 16 | hostname: oracle 17 | container_name: oracle 18 | restart: always 19 | ports: 20 | - "1521:1521" 21 | volumes: 22 | - ./init.d:/opt/oracle/scripts 23 | - data:/data 24 | - fra:/u01/app/oracle/fast_recovery_area 25 | environment: 26 | ORACLE_SID: MAIN 27 | ORACLE_PWD: oracle 28 | 29 | -------------------------------------------------------------------------------- /scripts/install_jmxterm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # jmxterm is a command line utility for accessing JMX Metrics; it is a great way to explore metrics of a Java application launched 5 | # within a container. 6 | # 7 | 8 | cd $(dirname $0)/.. 9 | 10 | if [ ! -f ./kafka/jmx_prometheus/jmxterm.jar ]; then 11 | 12 | echo "" 13 | echo "downloading jmxterm.jar" 14 | echo "" 15 | curl -s -L -o ./kafka/jmx_prometheus/jmxterm.jar https://github.com/jiaqi/jmxterm/releases/download/v1.0.2/jmxterm-1.0.2-uber.jar 16 | 17 | else 18 | 19 | echo "" 20 | echo "jmxterm.jar already downloaded" 21 | echo "" 22 | 23 | fi 24 | -------------------------------------------------------------------------------- /dashboards/superset/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/superset:latest 2 | 3 | USER root 4 | 5 | #RUN pip install apache-superset[druid] 6 | 7 | RUN pip install pydruid 8 | RUN pip install pydruid[cli] 9 | RUN pip install trino 10 | 11 | #RUN npm i @superset-ui/legacy-preset-chart-kepler 12 | 13 | USER superset 14 | 15 | COPY ./superset_config.py /app/pythonpath/superset_config.py 16 | 17 | RUN export FLASK_APP=superset 18 | RUN superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin 19 | RUN superset db upgrade 20 | RUN superset init 21 | 22 | #RUN superset load_examples 23 | 24 | -------------------------------------------------------------------------------- /scripts/volume_sizes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # only works on running containers, since volume size is based on executing the du command on all volumes from within the container 5 | # 6 | 7 | containers=$(docker ps --format '{{.Names}}|{{.ID}}' | sort) 8 | 9 | for container in $containers; do 10 | 11 | id=$(echo $container | cut -d\| -f2) 12 | name=$(echo $container | cut -d\| -f1) 13 | volumes=$(docker inspect -f '{{range .Mounts}}{{ if eq .Type "volume" }}{{.Destination}} {{end}}{{end}}' $id) 14 | 15 | echo $name 16 | 17 | if [ "$volumes" != "" ]; then 18 | docker exec $id du -d 0 -h $volumes 19 | fi 20 | 21 | echo "" 22 | 23 | done 24 | 25 | 26 | -------------------------------------------------------------------------------- /sqlserver/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | mssql: {} 9 | 10 | services: 11 | mssql: 12 | image: mcr.microsoft.com/mssql/server:${VERSION} 13 | platform: ${PLATFORM} 14 | hostname: mssql 15 | container_name: mssql 16 | # healthcheck: 17 | # test: pg_isready -U main 18 | # interval: 10s 19 | # timeout: 5s 20 | # retries: 5 21 | ports: 22 | - "1433:1433" 23 | volumes: 24 | - mssql:/var/opt/mssql 25 | - ./data:/data 26 | environment: 27 | ACCEPT_EULA: "Y" 28 | MSSQL_SA_PASSWORD: "MsSQL!Password" 29 | MSSQL_PID: Developer 30 | -------------------------------------------------------------------------------- /postgres/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | postgres: {} 9 | 10 | services: 11 | postgres: 12 | image: postgres:${VERSION} 13 | hostname: postgres 14 | container_name: postgres 15 | healthcheck: 16 | test: pg_isready -U main 17 | interval: 10s 18 | timeout: 5s 19 | retries: 5 20 | ports: 21 | - "5432:5432" 22 | command: "postgres -c wal_level=logical" 23 | volumes: 24 | - postgres:/var/lib/postgresql/data 25 | - ./data:/data 26 | environment: 27 | POSTGRES_PASSWORD: postgres 28 | POSTGRES_USER: admin 29 | POSTGRES_DB: main 30 | 31 | -------------------------------------------------------------------------------- /opensearch/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | services: 8 | opensearch: 9 | image: opensearchproject/opensearch:2.5.0 10 | container_name: opensearch 11 | environment: 12 | - "discovery.type=single-node" 13 | - "plugins.security.disabled=true" 14 | ports: 15 | - "9200:9200" 16 | 17 | opensearch-dashboards: 18 | image: opensearchproject/opensearch-dashboards:2.5.0 19 | container_name: opensearch-dashboards 20 | ports: 21 | - "5601:5601" 22 | environment: 23 | - "OPENSEARCH_HOSTS=http://opensearch:9200" 24 | - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" 25 | depends_on: 26 | - opensearch 27 | -------------------------------------------------------------------------------- /mysql5/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | mysql5: {} 9 | 10 | services: 11 | mysql5: 12 | platform: linux/amd64 13 | image: mysql:5.7.38 14 | hostname: mysql5 15 | container_name: mysql5 16 | ports: 17 | - "3307:3306" 18 | healthcheck: 19 | test: exit 0 20 | interval: 15s 21 | timeout: 5s 22 | retries: 30 23 | start_period: 10s 24 | volumes: 25 | - ./mysql.cnf:/etc/mysql/mysql.conf.d/mysql.cnf 26 | - ./init.d:/docker-entrypoint-initdb.d 27 | - ./data:/data 28 | environment: 29 | MYSQL_ROOT_PASSWORD: mysql 30 | MYSQL_USER: user 31 | MYSQL_PASSWORD: userpw 32 | 33 | -------------------------------------------------------------------------------- /druid/scripts/druid_download_extensions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname $0)/.. 4 | 5 | #if [ ! -f ./prometheus-emitter-0.22.0.jar ]; then 6 | # curl -s -O https://repo1.maven.org/maven2/org/apache/druid/extensions/contrib/prometheus-emitter/0.22.0/prometheus-emitter-0.22.0.jar 7 | #fi 8 | 9 | druid-protobuf-extension 10 | 11 | 12 | docker run -it --rm --entrypoint '' -v $(pwd)/druid/extensions:/opt/druid/extensions -v "druid_shared:/opt/shared" "apache/druid:0.22.0" sh -c "java -cp \"lib/*\" -Ddruid.extensions.directory=\"extensions\" -Ddruid.extensions.hadoopDependenciesDir=\"hadoop-dependencies\" org.apache.druid.cli.Main tools pull-deps --no-default-hadoop -c \"org.apache.druid.extensions.contrib:druid-protobuf-extension:0.22.0\"" 13 | 14 | 15 | -------------------------------------------------------------------------------- /bin/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | cd "$(dirname -- "$0")" 5 | 6 | cd .. 7 | 8 | (cd monitoring; docker compose down -v) 9 | (cd dashboards; docker compose down -v) 10 | (cd ksqlDB; docker compose down -v) 11 | (cd connect; docker compose down -v) 12 | (cd ksqlDB; docker compose down -v) 13 | (cd kafka; docker compose down -v) 14 | (cd kafka-raft; docker compose down -v) 15 | (cd mongo; docker compose down -v) 16 | (cd mysql; docker compose down -v) 17 | (cd mysql5; docker compose down -v) 18 | (cd postgres; docker compose down -v) 19 | (cd storage; docker compose down -v) 20 | (cd flink; docker compose down -v) 21 | (cd elasticsearch; docker compose down -v) 22 | (cd druid; docker compose down -v) 23 | (cd pinot; docker compose down -v) 24 | (cd cassandra; docker compose down -v) 25 | -------------------------------------------------------------------------------- /mysql/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | mysql: {} 9 | 10 | services: 11 | mysql: 12 | platform: ${PLATFORM} 13 | image: ${IMAGE} 14 | hostname: mysql 15 | container_name: mysql 16 | command: --default-authentication-plugin=mysql_native_password 17 | ports: 18 | - "3306:3306" 19 | healthcheck: 20 | test: exit 0 21 | interval: 15s 22 | timeout: 5s 23 | retries: 30 24 | start_period: 10s 25 | volumes: 26 | - mysql:/var/lib/mysql 27 | - ./cnf/local.cnf:/etc/mysql/conf.d/local.cnf 28 | - ./init.d:/docker-entrypoint-initdb.d 29 | - ./data:/data 30 | environment: 31 | MYSQL_ROOT_PASSWORD: mysql 32 | MYSQL_USER: user 33 | MYSQL_PASSWORD: userpw 34 | 35 | -------------------------------------------------------------------------------- /bin/connect-log: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | if ! [ -x "$(command -v jq)" ]; then 5 | echo "" 6 | echo "jq is not found, please install and make it available on your path, https://stedolan.github.io/jq" 7 | echo "" 8 | exit 9 | fi 10 | 11 | CONNECT_URL_1="http://localhost:18083" 12 | CONNECT_URL_2="http://localhost:28083" 13 | 14 | ACCEPT="Accept:application/json" 15 | CONTENT_TYPE="Content-Type:application/json" 16 | 17 | LOG=$1 18 | shift 19 | 20 | #LEVEL=$1 21 | #shift 22 | 23 | echo "Connector 1" 24 | curl -k -s -H "${ACCEPT}" -H "${CONTENT_TYPE}" -X PUT "${CONNECT_URL_1}/admin/loggers/${LOG}" -d '{"level": "TRACE"}' | jq '.' 25 | 26 | echo "Connector 2" 27 | curl -k -s -H "${ACCEPT}" -H "${CONTENT_TYPE}" -X PUT "${CONNECT_URL_2}/admin/loggers/${LOG}" -d '{"level": "TRACE"}' | jq '.' 28 | 29 | #plugin=$(cat ${CONFIG} | jq -r '."connector.class"') 30 | #shift 31 | -------------------------------------------------------------------------------- /dashboards/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | services: 8 | superset: 9 | build: ./superset 10 | container_name: superset 11 | healthcheck: 12 | test: wget --spider http://localhost:8088/superset/welcome/ || exit 1 13 | start_period: 3s 14 | interval: 3s 15 | timeout: 2s 16 | retries: 3 17 | ports: 18 | - 28088:8088 19 | environment: 20 | WTF_CSRF_ENABLED: "False" 21 | MAPBOX_API_KEY: "${MAPBOX_API_KEY:-}" 22 | 23 | turnilo: 24 | build: ./turnilo 25 | container_name: turnilo 26 | healthcheck: 27 | test: wget --spider http://localhost:9090 || exit 1 28 | start_period: 3s 29 | interval: 3s 30 | timeout: 2s 31 | retries: 3 32 | ports: 33 | - 9091:9090 34 | environment: 35 | DRUID: http://druid-router:8888 36 | 37 | -------------------------------------------------------------------------------- /pinot/configs/pinot-controller.conf: -------------------------------------------------------------------------------- 1 | # 2 | # ensure that any changes here are also reflected in the command line changes from the docker-compose, 3 | # when a config file is provided, it is unclear the priority of the properties supplied. 4 | # 5 | pinot.service.role=CONTROLLER 6 | pinot.cluster.name=Pinot 7 | pinot.zk.server=pinot-zookeeper:2181 8 | pinot.set.instance.id.to.hostname=true 9 | 10 | controller.port=9000 11 | controller.vip.host=pinot-controller 12 | controller.vip.port=9000 13 | controller.data.dir=/opt/pinot/data/pinotServerData 14 | # 15 | # The only way to adjust this property is to point to the config file 16 | # 17 | controller.disable.ingestion.groovy=false 18 | # 19 | 20 | realtime.segment.serverUploadToDeepStore = true 21 | 22 | pinot.controller.fs.factory.class.file=org.apache.pinot.spi.filesystem.LocalPinotFS 23 | pinot.controller.segment.store.uri=file:///opt/storage/pinotSegments 24 | 25 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-1.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-1:9094; 38 | } 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-2.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-2:9094; 38 | } 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-3.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-3:9094; 38 | } 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-4.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-4:9094; 38 | 39 | } 40 | 41 | } 42 | 43 | 44 | -------------------------------------------------------------------------------- /oracle/init.d/setup/logminer.sql: -------------------------------------------------------------------------------- 1 | 2 | SELECT SUPPLEMENTAL_LOG_DATA_MIN, SUPPLEMENTAL_LOG_DATA_ALL, FORCE_LOGGING FROM V$DATABASE; 3 | 4 | SHUTDOWN IMMEDIATE 5 | 6 | STARTUP MOUNT 7 | 8 | ALTER DATABASE ARCHIVELOG; 9 | 10 | ALTER DATABASE OPEN; 11 | 12 | ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; 13 | 14 | SELECT SUPPLEMENTAL_LOG_DATA_MIN, SUPPLEMENTAL_LOG_DATA_ALL, FORCE_LOGGING FROM V$DATABASE; 15 | 16 | ALTER SESSION SET "_ORACLE_SCRIPT"=true; 17 | 18 | CREATE ROLE LOGMNR_ROLE; 19 | GRANT CREATE SESSION TO LOGMNR_ROLE; 20 | GRANT EXECUTE_CATALOG_ROLE,SELECT ANY TRANSACTION ,SELECT ANY DICTIONARY TO LOGMNR_ROLE; 21 | 22 | CREATE USER C_USER IDENTIFIED BY C_PASS; 23 | GRANT LOGMNR_ROLE TO C_USER; 24 | ALTER USER C_USER QUOTA UNLIMITED ON USERS; 25 | 26 | ALTER DATABASE ADD SUPPLEMENTAL LOG DATA; 27 | ALTER SYSTEM ARCHIVE LOG CURRENT; 28 | 29 | ALTER SESSION SET CONTAINER = CDB$ROOT; 30 | GRANT SELECT ON SYS.USER$ TO C_USER; 31 | 32 | -------------------------------------------------------------------------------- /cassandra/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | cassandra_data: {} 8 | etc: {} 9 | 10 | services: 11 | cassandra: 12 | image: cassandra:4.0.4 13 | hostname: cassandra 14 | container_name: cassandra 15 | ports: 16 | - "9042:9042" 17 | healthcheck: 18 | test: ["CMD", "cqlsh", "-e", "describe keyspaces" ] 19 | interval: 5s 20 | timeout: 5s 21 | retries: 20 22 | start_period: 20s 23 | volumes: 24 | - cassandra_data:/var/lib/cassandra 25 | - etc:/etc/cassandra 26 | - ./data:/data 27 | environment: 28 | CASSANDRA_CLUSTER_NAME: cassandra 29 | CASSANDRA_DATACENTER: dc_main 30 | CASSANDRA_PASSWORD: cassandra 31 | # CASSANDRA_SEEDS: "cass1,cass2" 32 | # CASSANDRA_RACK: West 33 | # CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch 34 | # CASSANDRA_NUM_TOKENS: 128 35 | -------------------------------------------------------------------------------- /mongo/secret.key: -------------------------------------------------------------------------------- 1 | jLbzokH8drDlUwRiHsZbf09zqaFdZepD8pDJWPKPp9QQ8eEsF6jHO2dlByDylFZj 2 | 3dtgvPweXIUNx4ym52Hl5e0EBHZkVdfQudytXAmHFaePsFsa6PxLhz0MfqEvxoAx 3 | U6YnDAb/vln3DRlpxx6ALavDa2X1ZefyL+gEBWoCbNsrRVHrjegaR22FzLaO/VLt 4 | gTuobHcYtCbAlgMLyuT6yCZm4JLCAHxiubSsu3SRVeG3M8oxGPSCPC8lxngym3UD 5 | ZZLEdUmlC67QQVIn0kpcLfb2pjhjy4Ik8wtF0T5Z/RXYQQeYRiEXNLhzCdITKw/p 6 | s84ooZs+t/JcuT1Oug73qpwiJRMRNrEO62i0DFlCwY3sxQvfQ7yjH2tPun4qgVqQ 7 | gLgU/FwKGhRcBkCcQ+ZejZIrnd3J7WMQsAK6J5rY4pEH2dZq6H6CFHlvhk7+Dr5f 8 | olIcnjXglPTLcO0UmQhjck502bfljavdl8ke9aXBDRHMKHqN7tBbXJMuxAKoEJoT 9 | tDTjae+q3eKEhuCzm8yP79ahXnBj41egP0dpfKAFDcHPkGs1eIYeRX9G3/4tRFkn 10 | 4SnoSjDSQMjFpc9N3zWTMgEHRFAtNZ+URgruAOvMsieQBE2+jzT94oj1jIoP8sRZ 11 | yHr1FLF75HGqW9q0SgnFFAJsuSjDQVRHM3WfHCia9zIe/nbdGCk5THg8EikuE1gY 12 | 2hzleV4DriWemXT9t/zfV+8bKkcpQ++TCK9i8u6fm9S/yASQ2klO/Wv/cVm7+tDj 13 | LAY05BYIyzCwqt1fmXgk2C7uSxC5Wqitb7KE70HwE4A8hw53/Lhr+uC+mOBdXRaU 14 | R12mB5urCCz/M9Ue/QXmjwJgOuI1uHb303NFv7WoyjeherFwArh91nbLdW3xaQ0U 15 | Q8A29ZkcFoH0GAl8VYjoFtBfz21EpE8NFK9zKFCJLCAVwmscfru4lTIzBmp7UNge 16 | DmkjwQeKKxsqNmmtzCUov9sFkUuxucvjwiHpCaZwDk0JHG8X 17 | -------------------------------------------------------------------------------- /kafka-lb/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | services: 8 | 9 | broker-1-lb: 10 | # image: nginx:latest 11 | build: ./nginx 12 | container_name: broker-1-lb 13 | hostname: broker-1-lb 14 | cap_add: 15 | - NET_ADMIN 16 | volumes: 17 | - ./nginx/broker-1.conf:/etc/nginx/nginx.conf 18 | 19 | broker-2-lb: 20 | #image: nginx:latest 21 | build: ./nginx 22 | container_name: broker-2-lb 23 | hostname: broker-2-lb 24 | cap_add: 25 | - NET_ADMIN 26 | volumes: 27 | - ./nginx/broker-2.conf:/etc/nginx/nginx.conf 28 | 29 | broker-3-lb: 30 | #image: nginx:latest 31 | build: ./nginx 32 | container_name: broker-3-lb 33 | hostname: broker-3-lb 34 | cap_add: 35 | - NET_ADMIN 36 | volumes: 37 | - ./nginx/broker-3.conf:/etc/nginx/nginx.conf 38 | 39 | broker-4-lb: 40 | #image: nginx:latest 41 | build: ./nginx 42 | container_name: broker-4-lb 43 | hostname: broker-4-lb 44 | cap_add: 45 | - NET_ADMIN 46 | volumes: 47 | - ./nginx/broker-4.conf:/etc/nginx/nginx.conf 48 | 49 | -------------------------------------------------------------------------------- /elasticsearch/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | services: 8 | 9 | elasticsearch: 10 | image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION} 11 | hostname: elasticsearch 12 | container_name: elasticsearch 13 | ports: 14 | - 9200:9200 15 | - 9300:9300 16 | healthcheck: 17 | test: curl -s -f http://localhost:9200/_cat/health || exit 1 18 | interval: 15s 19 | timeout: 5s 20 | retries: 20 21 | start_period: 15s 22 | environment: 23 | xpack.security.enabled: "false" 24 | discovery.type: single-node 25 | ES_JAVA_OPTS: "-Xms512m -Xmx512m" 26 | 27 | kibana: 28 | image: docker.elastic.co/kibana/kibana:${VERSION} 29 | hostname: kibana 30 | container_name: kibana 31 | depends_on: 32 | - elasticsearch 33 | ports: 34 | - 5601:5601 35 | healthcheck: 36 | test: curl -s -f http://localhost:5601/login || exit 1 37 | interval: 15s 38 | timeout: 5s 39 | retries: 20 40 | start_period: 15s 41 | environment: 42 | xpack.security.enabled: "false" 43 | discovery.type: "single-node" 44 | -------------------------------------------------------------------------------- /bin/pinot: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 [-d|--delete] {schema|table} file" 6 | echo "" 7 | } 8 | 9 | if [ $# -lt 1 ]; then 10 | usage 11 | exit 12 | fi 13 | 14 | case $1 in 15 | -d|--delete) 16 | DELETE=true 17 | shift 18 | ;; 19 | *) 20 | DELETE=false 21 | ;; 22 | esac 23 | 24 | OPERATION=$1 25 | shift 26 | 27 | FIELD=$1 28 | shift 29 | 30 | URL=http://localhost:9000 31 | 32 | CONTENT_TYPE_HDR='Content-Type:application/json' 33 | 34 | 35 | # DATA=$1 36 | # if [ ! -f "$DATA" ]; then 37 | # echo "skipping $DATA, file not found." 38 | # return 39 | # fi 40 | 41 | case "${OPERATION}" in 42 | schema) 43 | if [ "$DELETE" == true ]; then 44 | curl -s -X DELETE -H $CONTENT_TYPE_HDR "$URL/schemas/$FIELD" | jq 45 | else 46 | curl -s -X POST -H $CONTENT_TYPE_HDR -d "@$FIELD" "$URL/schemas" | jq 47 | fi 48 | ;; 49 | table) 50 | if [ "$DELETE" == true ]; then 51 | curl -s -X DELETE -H $CONTENT_TYPE_HDR "$URL/tables/$FIELD" | jq 52 | else 53 | curl -s -X POST -H $CONTENT_TYPE_HDR -d "@$FIELD" "$URL/tables" | jq 54 | fi 55 | ;; 56 | *) 57 | usage 58 | esac 59 | -------------------------------------------------------------------------------- /storage/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | minio: {} 8 | 9 | services: 10 | 11 | minio: 12 | image: minio/minio:latest 13 | container_name: minio 14 | hostname: minio 15 | ports: 16 | - "9000:9000" 17 | - "9001:9001" 18 | command: server /data --console-address ":9001" 19 | healthcheck: 20 | test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] 21 | interval: 30s 22 | timeout: 20s 23 | retries: 3 24 | volumes: 25 | - minio:/data 26 | environment: 27 | - MINIO_ROOT_USER=admin 28 | - MINIO_ROOT_PASSWORD=miniominio 29 | - MINIO_SKIP_CLIENT=no 30 | - MINIO_REGION=us-east-1 31 | 32 | # minio-create-buckets: 33 | # image: minio/mc:latest 34 | # depends_on: 35 | # - minio 36 | # entrypoint: > 37 | # /bin/sh -c " 38 | # /usr/bin/mc config host add minio http://minio:9000 admin miniominio --api s3v4; 39 | # while ! /usr/bin/mc mb minio/public ; do echo 'waiting for minio'; sleep 1; done; 40 | # /usr/bin/mc policy set download minio/public; 41 | # exit 0; 42 | # " 43 | 44 | fake-gcs-server: 45 | image: fsouza/fake-gcs-server:1.37.12 46 | container_name: fake-gcs-server 47 | hostname: fake-gcs-server 48 | ports: 49 | - "8001:8001" 50 | volumes: 51 | - ./data/gcs:/data 52 | command: -port 8001 -scheme https -external-url https://fake-gcs-server:8001 53 | -------------------------------------------------------------------------------- /scripts/.zshrc: -------------------------------------------------------------------------------- 1 | # 2 | # Example aliases to consider for your local environment 3 | # 4 | 5 | alias d='docker' 6 | alias dc='docker compose' 7 | alias dps='docker ps --format "table {{.ID}}\t{{.Names}}\t{{.RunningFor}}\t{{.Status}}\t{{.Size}}\t{{.Ports}}"' 8 | 9 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 10 | 11 | alias kcc='kafka-console-consumer \ 12 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 13 | --key-deserializer=org.apache.kafka.common.serialization.BytesDeserializer \ 14 | --property print.key=true \ 15 | --property key.separator=" | " \ 16 | --from-beginning \ 17 | --topic ' 18 | 19 | alias kccv='kafka-console-consumer \ 20 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 21 | --from-beginning \ 22 | --topic' 23 | 24 | alias kacc='kafka-avro-console-consumer \ 25 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 26 | --property schema.registry.url="http://localhost:8081" \ 27 | --key-deserializer=org.apache.kafka.common.serialization.BytesDeserializer \ 28 | --property print.key=true \ 29 | --property key.separator=" | " \ 30 | --from-beginning --topic' 31 | 32 | alias kaccv='kafka-avro-console-consumer \ 33 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 34 | --property schema.registry.url="http://localhost:8081" \ 35 | --from-beginning --topic' 36 | -------------------------------------------------------------------------------- /scripts/stats.sh: -------------------------------------------------------------------------------- 1 | AWK=$(cat <= 1024**i || i == 0) { 43 | printf("%.2f %s\n", total/(1024**i), binaryunits[i]); 44 | break; 45 | } 46 | } 47 | } 48 | 49 | EOF 50 | ) 51 | 52 | 53 | SIZE=$(docker stats --no-stream --format "table {{.MemUsage}}" | tail -n +2 | cut -d/ -f1 | sed -E "s/([[:digit:]]+)([[:alpha:]]+)/\1 \2/" | awk "${AWK}") 54 | 55 | echo "" 56 | docker stats --no-stream 57 | echo "" 58 | echo "Total Memory Usage: $SIZE" 59 | echo "" 60 | -------------------------------------------------------------------------------- /pinot/configs/pinot-server.conf: -------------------------------------------------------------------------------- 1 | # 2 | # ensure that any changes here are also reflected in the command line changes from the docker-compose, 3 | # when a config file is provided, it is unclear the priority of the properties supplied. 4 | # 5 | pinot.service.role=SERVER 6 | pinot.cluster.name=Pinot 7 | pinot.zk.server=pinot-zookeeper:2181 8 | pinot.set.instance.id.to.hostname=true 9 | 10 | controller.restapi.protocol=http 11 | controller.restapi.host=pinot-controller 12 | controller.restapi.port=9000 13 | 14 | pinot.controller.host=pinot-controller 15 | pinot.controller.port=9000 16 | 17 | controller.host=pinot-controller 18 | controller.port=9000 19 | 20 | 21 | realtime.segment.serverUploadToDeepStore = true 22 | 23 | #pinot.server.segment.store.directory=/opt/storage 24 | #pinot.server.instance.enable.split.commit=true 25 | 26 | #pinot.server.instance.segment.store.uri=file://opt/storage 27 | #pinot.server.instance.enable.split.commit=true 28 | #pinot.server.fs.factory.class.local=org.apache.pinot.spi.filesystem.LocalPinotFS 29 | 30 | #pinot.server.segment.fetcher 31 | pinot.server.segment.fetcher.className=org.apache.pinot.spi.filesystem.LocalPinotFS 32 | pinot.server.segment.fetcher.protocol=file 33 | 34 | #pinot.server.instance.segmentTarDir=/opt/pinot/data/pinotSegments 35 | pinot.server.instance.segmentTarDir=/opt/storage/pinotSegments 36 | pinot.server.instance.dataDir=/opt/pinot/data/pinotServerData 37 | 38 | ##pinot.server.instance.segment.store.uri="file:///opt/storage" 39 | #pinot.server.instance.enable.split.commit=true 40 | #pinot.server.storage.factory.class.(scheme)= -------------------------------------------------------------------------------- /mysql5/mysql.cnf: -------------------------------------------------------------------------------- 1 | # For advice on how to change settings please see 2 | # http://dev.mysql.com/doc/refman/5.7/en/server-configuration-defaults.html 3 | 4 | [mysqld] 5 | # 6 | # Remove leading # and set to the amount of RAM for the most important data 7 | # cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%. 8 | # innodb_buffer_pool_size = 128M 9 | # 10 | # Remove leading # to turn on a very important data integrity option: logging 11 | # changes to the binary log between backups. 12 | # log_bin 13 | # 14 | # Remove leading # to set options mainly useful for reporting servers. 15 | # The server defaults are faster for transactions and fast SELECTs. 16 | # Adjust sizes as needed, experiment to find the optimal values. 17 | # join_buffer_size = 128M 18 | # sort_buffer_size = 2M 19 | # read_rnd_buffer_size = 2M 20 | skip-host-cache 21 | skip-name-resolve 22 | #datadir=/var/lib/mysql 23 | #socket=/var/lib/mysql/mysql.sock 24 | #secure-file-priv=/var/lib/mysql-files 25 | user=mysql 26 | 27 | # Disabling symbolic-links is recommended to prevent assorted security risks 28 | symbolic-links=0 29 | 30 | #log-error=/var/log/mysqld.log 31 | #pid-file=/var/run/mysqld/mysqld.pid 32 | 33 | # ---------------------------------------------- 34 | # Enable the binlog for replication & CDC 35 | # ---------------------------------------------- 36 | 37 | # Enable binary replication log and set the prefix, expiration, and log format. 38 | # The prefix is arbitrary, expiration can be short for integration tests but would 39 | # be longer on a production system. Row-level info is required for ingest to work. 40 | # Server ID is required, but this will vary on production systems 41 | server-id = 111111 42 | log_bin = mysql-bin 43 | expire_logs_days = 101 44 | #binlog_format = row 45 | 46 | -------------------------------------------------------------------------------- /flink/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | services: 8 | jobmanager: 9 | image: flink:${VERSION} 10 | container_name: flink_jobmanager 11 | ports: 12 | - "48081:8081" 13 | - "46123:6123" 14 | - "46124:6124" 15 | - "46125:6125" 16 | healthcheck: 17 | test: curl --fail http://localhost:8081/overview 18 | interval: 15s 19 | timeout: 5s 20 | retries: 30 21 | start_period: 0s 22 | command: jobmanager 23 | volumes: 24 | - ./jars:/jars 25 | environment: 26 | - | 27 | FLINK_PROPERTIES= 28 | jobmanager.rpc.address: jobmanager 29 | metrics.reporters: prom 30 | metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory 31 | metrics.reporter.prom.port: 7071 32 | rest.flamegraph.enabled: true 33 | 34 | taskmanager: 35 | image: flink:${VERSION} 36 | depends_on: 37 | - jobmanager 38 | command: taskmanager 39 | #scale: 2 40 | deploy: 41 | replicas: 2 42 | healthcheck: 43 | test: exit 0 44 | interval: 15s 45 | timeout: 5s 46 | retries: 30 47 | start_period: 5s 48 | volumes: 49 | - ./data:/data 50 | environment: 51 | - | 52 | FLINK_PROPERTIES= 53 | jobmanager.rpc.address: jobmanager 54 | taskmanager.numberOfTaskSlots: 2 55 | parallelism.default: 2 56 | taskmanager.memory.network.max: 268435460b 57 | taskmanager.memory.network.min: 268435460b 58 | taskmanager.memory.network.fraction: .4 59 | cluster.evenly-spread-out-slots: true 60 | metrics.reporters: prom 61 | metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory 62 | metrics.reporter.prom.port: 7071 63 | rest.flamegraph.enabled: true 64 | -------------------------------------------------------------------------------- /bin/druid: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 {load|unload|task} specification-file [specification-files]" 6 | echo "" 7 | } 8 | 9 | if [ $# -lt 2 ]; then 10 | usage 11 | exit 12 | fi 13 | 14 | OPERATION=$1 15 | shift 16 | 17 | declare -a FILES=( "$@" ) 18 | 19 | URL=http://localhost:48081 20 | 21 | CONTENT_TYPE_HDR='Content-Type:application/json' 22 | 23 | task() { 24 | 25 | DATA=$1 26 | if [ ! -f "$DATA" ]; then 27 | echo "skipping $DATA, file not found." 28 | return 29 | fi 30 | 31 | #curl -s -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/indexer/v1/task 32 | curl -s -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/indexer/v1/task | jq -r ".task" 33 | #curl -X POST -H "Content-Type:application/json" -d @./druid/compact.json http://localhost:48081/druid/indexer/v1/task 34 | } 35 | 36 | load() { 37 | 38 | DATA=$1 39 | if [ ! -f "$DATA" ]; then 40 | echo "skipping $DATA, file not found." 41 | return 42 | fi 43 | 44 | curl -s -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/indexer/v1/supervisor | jq -r ".id" 45 | } 46 | 47 | unload() { 48 | 49 | DATA=$1 50 | if [ ! -f "$DATA" ]; then 51 | echo "skipping $DATA, file not found." 52 | return 53 | fi 54 | 55 | spec=$(cat $DATA | jq -r ".spec.dataSchema.dataSource | select (.!=null)") 56 | 57 | if [ "${spec}" == "" ]; then 58 | echo "skipping $DATA, unable to extract dataSource from spec." 59 | return 60 | fi 61 | 62 | curl -s -X POST $URL/druid/indexer/v1/supervisor/${spec}/reset 63 | curl -s -X POST $URL/druid/indexer/v1/supervisor/${spec}/shutdown 64 | } 65 | 66 | case "${OPERATION}" in 67 | task) 68 | for i in "${FILES[@]}"; do 69 | task $i 70 | done 71 | ;; 72 | load) 73 | for i in "${FILES[@]}"; do 74 | load $i 75 | done 76 | ;; 77 | unload) 78 | for i in "${FILES[@]}"; do 79 | unload $i 80 | done 81 | ;; 82 | *) 83 | usage 84 | esac 85 | -------------------------------------------------------------------------------- /druid/scripts/druid_dump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 {-mm|-h} [-i=interval} datasource" 6 | echo "" 7 | } 8 | 9 | SOURCE=h 10 | 11 | interval="" 12 | 13 | for i in "$@"; do 14 | case $i in 15 | --middlemanager|-m|-mm) 16 | SOURCE="mm" 17 | ;; 18 | --historical|-h) 19 | SOURCE="h" 20 | ;; 21 | --interval=*|-i=*) 22 | interval="${i#*=}" 23 | ;; 24 | *) 25 | datasource=$i 26 | ;; 27 | esac 28 | shift 29 | done 30 | 31 | if [ -z "${datasource}" ]; then 32 | usage 33 | exit 34 | fi 35 | 36 | 37 | echo "" 38 | echo $interval 39 | echo "" 40 | 41 | if [ "${SOURCE}" == "mm" ]; then 42 | 43 | docker exec -it druid-middlemanager sh -c "/opt/staging/scripts/export_rt.sh ${datasource}" 44 | 45 | # docker exec -it druid-middlemanager sh -c 'cd /opt/staging; FILE=$(find /opt/druid/var/druid/task -name 00000.smoosh -print -quit) && java -classpath "/opt/druid/lib/*" -Ddruid.extensions.loadList="[]" org.apache.druid.cli.Main tools dump-segment --directory $(dirname $FILE) --out ./x.txt ${COMMAND} 2>/dev/null && cat ./x.txt' 46 | 47 | else 48 | 49 | docker exec -it druid-historical sh -c "/opt/staging/scripts/export_historical.sh ${datasource} ${interval}" 50 | #docker exec -it druid-historical sh -c "/opt/staging/scripts/export_historical.sh ${datasource} 2021-12-05T13:00:00.000Z" 51 | 52 | fi 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | #docker run -it --rm --entrypoint '' -v "druid_shared:/opt/shared" "apache/druid:0.22.0" sh -c 'cd /tmp; FILE=$(find /opt/shared -name index.zip -print -quit) && unzip $FILE >/dev/null && java -classpath "/opt/druid/lib/*" -Ddruid.extensions.loadList="[]" org.apache.druid.cli.Main tools dump-segment --directory . --out ./x.txt --time-iso8601 --dump bitmaps --decompress-bitmaps 2>/dev/null && cat ./x.txt' 61 | #docker run -it --rm --entrypoint '' -v "druid_shared:/opt/shared" "apache/druid:0.22.0" sh -c "find /opt/shared -name index.zip -exec xargs -L1 echo \;" 62 | -------------------------------------------------------------------------------- /mongo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | db: {} 8 | configdb: {} 9 | 10 | services: 11 | mongo: 12 | image: mongo:5.0.9 13 | hostname: mongo 14 | container_name: mongo 15 | ports: 16 | - "27017:27017" 17 | # replica set is required for debezium to leverage CDC. 18 | # secret key is required due to authentication w/ replica sets (as key would need to be identical on all replacs) 19 | command: [ "--keyFile", "/secret.key", "--replSet", "rs0" ] 20 | # to use debezium, a replica-set is needed; and a replica-set has to be initialized; leverage health-check 21 | # to do this. Product requires at least 3 instances for a proper replica deployment, but for dev environment, 22 | # it can be enabled as a single instance 23 | healthcheck: 24 | test: test $$(echo "rs.status().ok || rs.initiate().ok" | mongo -u $${MONGO_INITDB_ROOT_USERNAME} -p $${MONGO_INITDB_ROOT_PASSWORD} --quiet) -eq 1 25 | interval: 10s 26 | start_period: 30s 27 | volumes: 28 | - db:/data/db 29 | - configdb:/data/configdb 30 | - ./secret.key:/secret.key 31 | - ./db-init/:/docker-entrypoint-initdb.d 32 | environment: 33 | MONGO_INITDB_ROOT_USERNAME: root 34 | MONGO_INITDB_ROOT_PASSWORD: mongo 35 | MONGO_INITDB_DATABASE: main 36 | MONGO_USER: mongo_user 37 | MONGO_PASSWORD: mongo_password 38 | 39 | mongo-express: 40 | image: mongo-express:0.54.0 41 | container_name: mongo-express 42 | hostname: mongo-express 43 | ports: 44 | - 28081:8081 45 | healthcheck: 46 | test: wget --spider http://localhost:8081/ || exit 1 47 | interval: 10s 48 | start_period: 20s 49 | environment: 50 | #ME_CONFIG_MONGODB_SERVER: '' 51 | ME_CONFIG_MONGODB_URL: mongodb://mongo:27017 52 | ME_CONFIG_MONGODB_ADMINUSERNAME: root 53 | ME_CONFIG_MONGODB_ADMINPASSWORD: mongo 54 | #ME_CONFIG_BASICAUTH_USERNAME: admin 55 | #ME_CONFIG_BASICAUTH_PASSWORD: mongo-express 56 | -------------------------------------------------------------------------------- /ksqlDB/jmx_prometheus/streams-config.yml: -------------------------------------------------------------------------------- 1 | lowercaseOutputName: true 2 | rules: 3 | - pattern: java.lang 4 | #- pattern: application(.*) 5 | #- pattern: application 6 | #- pattern: kafka.streams.* 7 | - pattern: "application<>(.+): (.+)__(.+)" 8 | name: application_streams 9 | value: 1 10 | labels: 11 | task_id: $1 12 | sources: $2 13 | sinks: $3 14 | - pattern: kafka.streams<>(.+):.+ 15 | name: kafka_streams_stream_record_cache_metrics 16 | labels: 17 | thread_id: "$1" 18 | task_id: "$2" 19 | partition_id: "$3" 20 | record-cache-id: "$4" 21 | metric: "$5" 22 | - pattern: kafka.streams<>(.+):.+ 23 | name: kafka_streams_stream_state_metrics 24 | labels: 25 | thread_id: "$1" 26 | task_id: "$2" 27 | partition_id: "$3" 28 | store_type: "$4" 29 | state_id: "$5" 30 | metric: "$6" 31 | - pattern: kafka.streams<>(.+):.+ 32 | name: kafka_streams_stream_thread_metrics_$2 33 | labels: 34 | thread_id: "$1" 35 | - pattern: kafka.streams<>(.+):.+ 36 | name: kafka_streams_stream_task_metrics_$4 37 | labels: 38 | thread_id: "$1" 39 | task_id: "$2" 40 | partition_id: "$3" 41 | - pattern: kafka.streams<>(.+):.+ 42 | name: kafka_streams_stream_processor_node_metrics_$5 43 | labels: 44 | thread_id: "$1" 45 | task_id: "$2" 46 | partition_id: "$3" 47 | processor_node_id: "$4" 48 | - pattern: kafka.producer<>(.+):.+ 49 | name: kafka_producer_producer_metrics_$2 50 | labels: 51 | client_id: "$1" 52 | - pattern: kafka.consumer<>(.+):.+ 53 | name: kafka_consumer_consumer_metrics_$2 54 | labels: 55 | client_id: "$1" 56 | #- pattern: kafka.(.*) 57 | 58 | 59 | -------------------------------------------------------------------------------- /oracle/README.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | 3 | * Oracle does not provide preconfigured docker images, however, it does provide a nice open-source project you can use to build them. 4 | 5 | * In order to build an Oracle Image you need 2 things 6 | 7 | 1. The Oracle installation for the specific oracle version you need from [Oracle Database Software Downloads](https://www.oracle.com/database/technologies/oracle-database-software-downloads.html). 8 | An account with oracle is needed to download the installation files (free). 9 | 10 | 2. The Oracle's GitHub [Docker Images](https://github.com/oracle/docker-images.git) project. 11 | 12 | * Once you have the database software and GitHub docker image repository 13 | 14 | * Got to the single instance dockerfiles for the oracle builds 15 | 16 | ``` 17 | cd OracleDatabase/SingleInstance/dockerfiles 18 | ``` 19 | 20 | * copy the database software into the version directory 21 | 22 | * build the docker image 23 | 24 | ``` 25 | buildDockerImage.sh -s -i -v {version} 26 | ``` 27 | 28 | * I have used 19.3.0, 18.4.0, and 12.2.0.1 standard edition with LogMiner enabled. 29 | 30 | * The scripts to enable LogMiner should work for each version, I haven't doubled-check this though with the script that ended up. 31 | This script to enable logminer is in init.d/startup/logminer.sql. 32 | 33 | * When the oracle instance is started for the first time `init.d/setup/logminer.sql` will execute after the database is built. 34 | The initial startup of oracle will take some time, since buildDockerImage.sh builds an image with all the software installed, but the database is not yet created. 35 | 36 | * Avoid doing `docker-compose down` on oracle, otherwise it will have to rebuild the database. You want to `stop` the database so the container image 37 | is not destored 38 | 39 | * On initial startup or any other startup, all SQL scripts in `init.d/startup` is executed; in alphabetical order. Write scripts 40 | where they are idempotent (trap exceptions in PSQL; e.g. table already existing) or truncate data before insertion. 41 | 42 | * Logminer notes 43 | 44 | * While I have used logminer with Oracle 11 XE, that was not based on Oracle's build scripts. 45 | 46 | * Logminer for 12+ requires standard or enterprise, express edition does not support logminer 47 | 48 | * Logminer in 11 was with XE and was with slightly different script. 49 | -------------------------------------------------------------------------------- /scripts/container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # This local environment was designed for a 64GB machine, but should work fine with 32GB. 5 | # 6 | # However, to run with fewer resources, certain services might be best to not be enabled. 7 | # 8 | # 9 | 10 | DIR=$(dirname $0)/.. 11 | 12 | DOCKER_COMPOSE=${DIR}/kafka/docker-compose.yml 13 | 14 | 15 | usage() { 16 | echo "" 17 | echo "Usage: $0 {--uncomment|--comment|--toggle} {broker-4|connect-2|connect-3|ksql-server-2}" 18 | echo "" 19 | } 20 | 21 | 22 | if [[ $# -eq 0 ]]; then 23 | usage 24 | exit 25 | fi 26 | 27 | COMMENT=toggle 28 | 29 | for i in "$@"; do 30 | case $i in 31 | --uncomment) 32 | COMMAND=uncomment 33 | ;; 34 | --comment) 35 | COMMAND=comment 36 | ;; 37 | --toggle) 38 | COMMAND=toggle 39 | ;; 40 | broker-4) 41 | DOCKER_COMPOSE=${DIR}/kafka/docker-compose.yml 42 | start=$(grep -n "^#\? broker-4:" ${DOCKER_COMPOSE} | cut -d: -f1) 43 | end=$(expr $(grep -n "^#\? schema-registry:" ${DOCKER_COMPOSE} | cut -d: -f1) - 2) 44 | ;; 45 | connect-2) 46 | DOCKER_COMPOSE=${DIR}/connect/docker-compose.yml 47 | start=$(grep -n "^#\? connect-2:" ${DOCKER_COMPOSE} | cut -d: -f1) 48 | end=$(expr $(grep -n "^#\? connect-3:" ${DOCKER_COMPOSE} | cut -d: -f1) - 2) 49 | ;; 50 | connect-3) 51 | DOCKER_COMPOSE=${DIR}/connect/docker-compose.yml 52 | start=$(grep -n "^#\? connect-3:" ${DOCKER_COMPOSE} | cut -d: -f1) 53 | end="$" 54 | ;; 55 | ksql-server-2) 56 | DOCKER_COMPOSE=${DIR}/ksqldb/docker-compose.yml 57 | start=$(grep -n "^#\? ksql-server-2:" ${DOCKER_COMPOSE} | cut -d: -f1) 58 | end="$" 59 | ;; 60 | *) 61 | usage 62 | ;; 63 | esac 64 | shift 65 | done 66 | 67 | 68 | 69 | IS_COMMENTED=$(sed -n "${start}p" ${DOCKER_COMPOSE} | cut -c1) 70 | 71 | if [[ "${IS_COMMENTED}" == "#" ]] && [[ "${COMMAND}" == "comment" ]]; then 72 | echo "" 73 | echo "already commented out" 74 | echo "" 75 | exit 76 | elif [[ "${IS_COMMENTED}" == " " ]] && [[ "${COMMAND}" == "uncomment" ]]; then 77 | echo "" 78 | echo "already uncommented" 79 | echo "" 80 | exit 81 | fi 82 | 83 | 84 | 85 | if [ "${IS_COMMENTED}" == "#" ]; then 86 | echo "uncommenting" 87 | sed -I "" -E -e "${start},${end}s/^#(.*)$/\1/" ${DOCKER_COMPOSE} 88 | else 89 | echo "commenting out" 90 | sed -I "" -E -e "${start},${end}s/^(.*)$/#\1/" ${DOCKER_COMPOSE} 91 | fi 92 | 93 | -------------------------------------------------------------------------------- /bin/ksql-restart: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # 'compose' is now a command built into Docker Desktop, use that is avaiable, other default to docker-compose script. 5 | # 6 | 7 | docker compose --help | grep -e "^Docker Compose$" 2>&1 >/dev/null 8 | if [ $? -eq 0 ]; then 9 | COMPOSE="docker compose" 10 | else 11 | COMPOSE="docker-compose" 12 | fi 13 | 14 | DIR=$(dirname $0) 15 | KSQLDB_DIR="${DIR}/../ksqlDB" 16 | KSQLDB_DOCKER_COMPOSE_FILE="${KSQLDB_DIR}/docker-compose.yml" 17 | CONTAINER=ksql-server 18 | 19 | usage() { 20 | echo "" 21 | echo "Usage: $0 {--eos|--alo|--kill|--shutdown|--restart}" 22 | echo "" 23 | } 24 | 25 | if [[ $# -eq 0 ]] || [[ $# -gt 5 ]]; then 26 | usage 27 | exit 28 | fi 29 | 30 | 31 | for i in "$@"; do 32 | case $i in 33 | -e|--eos|--exactly-once) 34 | eos=true 35 | ;; 36 | -a|--alo|--at-least-once) 37 | alo=true 38 | ;; 39 | -k|--kill) 40 | kill=true 41 | ;; 42 | -s|--shutdown) 43 | shutdown=true 44 | ;; 45 | -r|--restart) 46 | restart=true 47 | ;; 48 | *) 49 | ;; 50 | esac 51 | shift 52 | done 53 | 54 | 55 | if [[ "$eos" == true ]] && [[ "$alo" == true ]]; then 56 | usage 57 | echo " exactly_once (--eos, -e) and at_least_once (--alo, -a) are mutually exclusive" 58 | echo "" 59 | exit 60 | fi 61 | 62 | if [[ "$kill" == true ]] && [[ "$shutdown" == true ]]; then 63 | usage 64 | echo " shutdown (--shutdown, -s) and kill (--kill, -k) are mutually exclusive" 65 | echo "" 66 | exit 67 | fi 68 | 69 | # 70 | # alter the processing guarantee before restarting/killing 71 | # 72 | 73 | if [[ "$eos" == true ]]; then 74 | sed -I .bak -E "s/KSQL_KSQL_STREAMS_PROCESSING_GUARANTEE: (.*)/KSQL_KSQL_STREAMS_PROCESSING_GUARANTEE: exactly_once/" ${KSQLDB_DOCKER_COMPOSE_FILE} 75 | echo "" 76 | echo "docker-compose.yml updated for $CONTAINER, processing guarantee set to exactly_once" 77 | echo "" 78 | elif [[ "$alo" == true ]]; then 79 | sed -I .bak -E "s/KSQL_KSQL_STREAMS_PROCESSING_GUARANTEE: (.*)/KSQL_KSQL_STREAMS_PROCESSING_GUARANTEE: at_least_once/" ${KSQLDB_DOCKER_COMPOSE_FILE} 80 | echo "" 81 | echo "docker-compose.yml updated for $CONTAINER, processing guarantee set to at_least_once" 82 | echo "" 83 | fi 84 | 85 | 86 | # 87 | # shutdown/kill 88 | # 89 | 90 | if [ "$CONTAINER" == "$(docker ps --format "{{.Names}}" | grep $CONTAINER)" ]; then 91 | echo "" 92 | docker exec -it -e COLUMNS="`tput cols`" -e LINES="`tput lines`" $CONTAINER sh -c "ps -f -g 1" 93 | echo "" 94 | if [[ "$kill" == true ]]; then 95 | echo "killing $CONTAINER" 96 | docker kill $CONTAINER 97 | elif [[ "$shutdown" == true ]]; then 98 | echo "shutting down $CONTAINER" 99 | (cd $KSQLDB_DIR; $COMPOSE stop ksql-server) 100 | else 101 | echo "not shutting down or killing $CONTAINER" 102 | fi 103 | else 104 | echo "" 105 | if [ "$CONTAINER" == "$(docker ps -a --format "{{.Names}}" | grep $CONTAINER)" ]; then 106 | echo "${CONTAINER} is already stopped." 107 | else 108 | echo "${CONTAINER} does not exist (it is not running or stopped)." 109 | fi 110 | echo "" 111 | fi 112 | 113 | 114 | # 115 | # restart 116 | # 117 | 118 | if [[ "$restart" == true ]]; then 119 | (cd $KSQLDB_DIR; $COMPOSE up -d ksql-server) 120 | else 121 | echo "" 122 | echo "--restart not provided, container not restarted" 123 | echo "" 124 | fi 125 | -------------------------------------------------------------------------------- /monitoring/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | 4 | scrape_configs: 5 | 6 | - job_name: kafka 7 | #scrape_interval: 1s 8 | static_configs: 9 | - targets: 10 | - controller-0:7071 11 | - controller-1:7071 12 | - controller-2:7071 13 | - broker-1:7071 14 | - broker-2:7071 15 | - broker-3:7071 16 | - broker-4:7071 17 | labels: 18 | cluster_type: "kafka" 19 | cluster_id: "kafka-core" 20 | relabel_configs: 21 | - source_labels: [__address__] 22 | regex: "(.*):(.*)" 23 | target_label: instance 24 | replacement: "$1" 25 | - job_name: schema-registry 26 | #scrape_interval: 1s 27 | static_configs: 28 | - targets: 29 | - schema-registry:7071 30 | labels: 31 | cluster_type: "schema-registry" 32 | cluster_id: "schema-registry" 33 | relabel_configs: 34 | - source_labels: [__address__] 35 | regex: "(.*):(.*)" 36 | target_label: instance 37 | replacement: "$1" 38 | - job_name: kafka-connect 39 | #scrape_interval: 1s 40 | static_configs: 41 | - targets: 42 | - connect-1:7071 43 | - connect-2:7071 44 | - connect-3:7071 45 | labels: 46 | cluster_type: "connect" 47 | cluster_id: "connect-cluster" 48 | relabel_configs: 49 | - source_labels: [__address__] 50 | regex: "(.*):(.*)" 51 | target_label: instance 52 | replacement: "$1" 53 | - job_name: ksql 54 | #scrape_interval: 1s 55 | static_configs: 56 | - targets: 57 | - ksql-server:7071 58 | - ksql-server-2:7071 59 | labels: 60 | cluster_type: "streams" 61 | cluster_id: "ksql-cluster" 62 | relabel_configs: 63 | - source_labels: [__address__] 64 | regex: "(.*):(.*)" 65 | target_label: instance 66 | replacement: "$1" 67 | - job_name: kafka_exporter 68 | static_configs: 69 | - targets: 70 | - kafka-exporter:9308 71 | - job_name: druid-exporter 72 | static_configs: 73 | - targets: 74 | - druid-exporter:7071 75 | labels: 76 | cluster_type: "druid" 77 | cluster_id: "druid-exporter" 78 | relabel_configs: 79 | - source_labels: [__address__] 80 | regex: "(.*):(.*)" 81 | target_label: instance 82 | replacement: "$1" 83 | - job_name: druid 84 | static_configs: 85 | - targets: 86 | - druid-coordinator:7071 87 | - druid-router:7071 88 | - druid-broker:7071 89 | - druid-historical:7071 90 | - druid-middlemanager:7071 91 | labels: 92 | cluster_type: "druid" 93 | cluster_id: "druid" 94 | relabel_configs: 95 | - source_labels: [__address__] 96 | regex: "(.*):(.*)" 97 | target_label: instance 98 | replacement: "$1" 99 | - job_name: flink 100 | static_configs: 101 | - targets: 102 | - flink_jobmanager:7071 103 | - dev-flink-taskmanager-1:7071 104 | - dev-flink-taskmanager-2:7071 105 | labels: 106 | cluster_type: "flink" 107 | cluster_id: "flink" 108 | - job_name: targets 109 | file_sd_configs: 110 | - files: 111 | - /etc/prometheus/targets.json 112 | -------------------------------------------------------------------------------- /kafka/jmx_prometheus/kafka-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | rules: 4 | - pattern: java.lang 5 | - pattern: "kafka.server<>(.*): (.*)" 6 | name: kafka_server_app_info_$2 7 | labels: 8 | node_id: "$1" 9 | kafka_version: "$3" 10 | value: 1 11 | - pattern: "kafka.server<>current-state: (.+)" 12 | name: kafka_server_raft_metrics_current_state 13 | labels: 14 | state: $1 15 | value: 1 16 | - pattern: "kafka.server<>(.+): (.+)" 17 | - pattern: kafka.cluster<>Value 18 | - pattern: kafka.controller<>Value 19 | - pattern: kafka.controller<>Count 20 | - pattern: kafka.controller<>OneMinuteRate 21 | - pattern: kafka.log<>Value 22 | - pattern: kafka.network<>999thPercentile 23 | - pattern: kafka.network<>99thPercentile 24 | - pattern: kafka.network<>95thPercentile 25 | - pattern: kafka.network<>75thPercentile 26 | - pattern: kafka.network<>50thPercentile 27 | - pattern: kafka.server<>Value 28 | - pattern: kafka.server<>Value 29 | - pattern: kafka.server<>Value 30 | - pattern: kafka.server<>Count 31 | - pattern: kafka.server<>OneMinuteRate 32 | - pattern: kafka.server<>Value 33 | - pattern: kafka.server<>Count 34 | - pattern: kafka.server<>OneMinuteRate 35 | - pattern: kafka.server<>Count 36 | - pattern: kafka.server<>OneMinuteRate 37 | - pattern: kafka.server<>Count 38 | - pattern: kafka.server<>OneMinuteRate 39 | - pattern: kafka.server<>Count 40 | - pattern: kafka.server<>OneMinuteRate 41 | - pattern: kafka.server<>Count 42 | - pattern: kafka.server<>OneMinuteRate 43 | - pattern: kafka.server<>Count 44 | - pattern: kafka.server<>OneMinuteRate 45 | - pattern: kafka.server<>Count 46 | - pattern: kafka.server<>OneMinuteRate 47 | - pattern: kafka.server<>Count 48 | - pattern: kafka.server<>OneMinuteRate 49 | - pattern: kafka.server<>connections 50 | - pattern: kafka.server<>(.+) 51 | - pattern: kafka.serverValue 52 | 53 | -------------------------------------------------------------------------------- /kafka-raft/jmx_prometheus/kafka-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | rules: 4 | - pattern: java.lang 5 | - pattern: "kafka.server<>(.*): (.*)" 6 | name: kafka_server_app_info_$2 7 | labels: 8 | node_id: "$1" 9 | kafka_version: "$3" 10 | value: 1 11 | - pattern: "kafka.server<>current-state: (.+)" 12 | name: kafka_server_raft_metrics_current_state 13 | labels: 14 | state: $1 15 | value: 1 16 | - pattern: "kafka.server<>(.+): (.+)" 17 | - pattern: kafka.cluster<>Value 18 | - pattern: kafka.controller<>Value 19 | - pattern: kafka.controller<>Count 20 | - pattern: kafka.controller<>OneMinuteRate 21 | - pattern: kafka.log<>Value 22 | - pattern: kafka.network<>999thPercentile 23 | - pattern: kafka.network<>99thPercentile 24 | - pattern: kafka.network<>95thPercentile 25 | - pattern: kafka.network<>75thPercentile 26 | - pattern: kafka.network<>50thPercentile 27 | - pattern: kafka.server<>Value 28 | - pattern: kafka.server<>Value 29 | - pattern: kafka.server<>Value 30 | - pattern: kafka.server<>Count 31 | - pattern: kafka.server<>OneMinuteRate 32 | - pattern: kafka.server<>Value 33 | - pattern: kafka.server<>Count 34 | - pattern: kafka.server<>OneMinuteRate 35 | - pattern: kafka.server<>Count 36 | - pattern: kafka.server<>OneMinuteRate 37 | - pattern: kafka.server<>Count 38 | - pattern: kafka.server<>OneMinuteRate 39 | - pattern: kafka.server<>Count 40 | - pattern: kafka.server<>OneMinuteRate 41 | - pattern: kafka.server<>Count 42 | - pattern: kafka.server<>OneMinuteRate 43 | - pattern: kafka.server<>Count 44 | - pattern: kafka.server<>OneMinuteRate 45 | - pattern: kafka.server<>Count 46 | - pattern: kafka.server<>OneMinuteRate 47 | - pattern: kafka.server<>Count 48 | - pattern: kafka.server<>OneMinuteRate 49 | - pattern: kafka.server<>connections 50 | - pattern: kafka.server<>(.+) 51 | - pattern: kafka.serverValue 52 | 53 | -------------------------------------------------------------------------------- /kafka-zk/jmx_prometheus/kafka-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | rules: 4 | #- pattern: .* 5 | - pattern: java.lang 6 | - pattern: "kafka.server<>(.*): (.*)" 7 | name: kafka_server_app_info_$2 8 | labels: 9 | node_id: "$1" 10 | kafka_version: "$3" 11 | value: 1 12 | - pattern: "kafka.server<>current-state: (.+)" 13 | name: kafka_server_raft_metrics_current_state 14 | labels: 15 | state: $1 16 | value: 1 17 | - pattern: "kafka.server<>(.+): (.+)" 18 | - pattern: kafka.cluster<>Value 19 | - pattern: kafka.controller<>Value 20 | - pattern: kafka.controller<>Value 21 | - pattern: kafka.controller<>Count 22 | - pattern: kafka.controller<>OneMinuteRate 23 | - pattern: kafka.log<>Value 24 | - pattern: kafka.network<>999thPercentile 25 | - pattern: kafka.network<>99thPercentile 26 | - pattern: kafka.network<>95thPercentile 27 | - pattern: kafka.network<>75thPercentile 28 | - pattern: kafka.network<>50thPercentile 29 | - pattern: kafka.server<>Value 30 | - pattern: kafka.server<>Value 31 | - pattern: kafka.server<>Value 32 | - pattern: kafka.server<>Count 33 | - pattern: kafka.server<>OneMinuteRate 34 | - pattern: kafka.server<>Value 35 | - pattern: kafka.server<>Count 36 | - pattern: kafka.server<>OneMinuteRate 37 | - pattern: kafka.server<>Count 38 | - pattern: kafka.server<>OneMinuteRate 39 | - pattern: kafka.server<>Count 40 | - pattern: kafka.server<>OneMinuteRate 41 | - pattern: kafka.server<>Count 42 | - pattern: kafka.server<>OneMinuteRate 43 | - pattern: kafka.server<>Count 44 | - pattern: kafka.server<>OneMinuteRate 45 | - pattern: kafka.server<>Count 46 | - pattern: kafka.server<>OneMinuteRate 47 | - pattern: kafka.server<>Count 48 | - pattern: kafka.server<>OneMinuteRate 49 | - pattern: kafka.server<>Count 50 | - pattern: kafka.server<>OneMinuteRate 51 | - pattern: kafka.server<>connections 52 | - pattern: kafka.server<>(.+) 53 | - pattern: kafka.serverValue 54 | -------------------------------------------------------------------------------- /pinot/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | zk: {} 8 | zk_datalog: {} 9 | zk_logs: {} 10 | pinot_controller_data: {} 11 | pinot_broker_data: {} 12 | pinot_server_data: {} 13 | pinot_local_storage: {} 14 | 15 | services: 16 | pinot-zookeeper: 17 | image: zookeeper:3.5.8 18 | hostname: pinot-zookeeper 19 | container_name: pinot-zookeeper 20 | healthcheck: 21 | test: echo ruok | nc -w 2 localhost 2181 | grep imok 22 | start_period: 5s 23 | interval: 5s 24 | timeout: 5s 25 | retries: 3 26 | volumes: 27 | - zk:/data 28 | - zk_datalog:/datalog 29 | - zk_logs:/logs 30 | environment: 31 | ZOOKEEPER_CLIENT_PORT: 2181 32 | ZOOKEEPER_TICK_TIME: 2000 33 | ZOO_4LW_COMMANDS_WHITELIST: "ruok,stat" 34 | 35 | pinot-server: 36 | image: apachepinot/pinot:${VERSION} 37 | hostname: pinot-server 38 | container_name: pinot-server 39 | command: "StartServer -clusterName 'Pinot' -zkAddress pinot-zookeeper:2181 -config /opt/pinot/configs/pinot-server.conf" 40 | #command: "StartServer -clusterName 'Pinot' -zkAddress pinot-zookeeper:2181 -segmentDir /opt/pinot/data/pinotSegments -dataDir /opt/pinot/data/pinotServerData" 41 | restart: unless-stopped 42 | healthcheck: 43 | test: curl --fail http://localhost:8097/health || exit 1 44 | start_period: 5s 45 | interval: 5s 46 | timeout: 5s 47 | retries: 3 48 | depends_on: 49 | pinot-controller: 50 | condition: service_healthy 51 | volumes: 52 | - pinot_server_data:/opt/pinot/data 53 | - pinot_local_storage:/opt/storage 54 | - ./configs:/opt/pinot/configs 55 | environment: 56 | LOG4J_CONSOLE_LEVEL: DEBUG 57 | JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log" 58 | 59 | pinot-controller: 60 | image: apachepinot/pinot:${VERSION} 61 | hostname: pinot-controller 62 | container_name: pinot-controller 63 | command: "StartController -clusterName 'Pinot' -zkAddress pinot-zookeeper:2181 -config /opt/pinot/configs/pinot-controller.conf" 64 | #command: "StartController -clusterName 'Pinot' -zkAddress pinot-zookeeper:2181 -dataDir /opt/pinot/data/pinotServerData" 65 | restart: unless-stopped 66 | healthcheck: 67 | test: curl --fail http://localhost:9000/health || exit 1 68 | start_period: 5s 69 | interval: 5s 70 | timeout: 5s 71 | retries: 3 72 | depends_on: 73 | pinot-zookeeper: 74 | condition: service_healthy 75 | ports: 76 | - "9000:9000" 77 | volumes: 78 | - pinot_controller_data:/opt/pinot/data 79 | - pinot_local_storage:/opt/storage 80 | - ./configs:/opt/pinot/configs 81 | environment: 82 | LOG4J_CONSOLE_LEVEL: DEBUG 83 | JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log" 84 | 85 | pinot-broker: 86 | image: apachepinot/pinot:${VERSION} 87 | hostname: pinot-broker 88 | container_name: pinot-broker 89 | command: "StartBroker -clusterName 'Pinot' -zkAddress pinot-zookeeper:2181" 90 | restart: unless-stopped 91 | healthcheck: 92 | test: curl --fail http://localhost:8099/health || exit 1 93 | start_period: 5s 94 | interval: 5s 95 | timeout: 5s 96 | retries: 3 97 | depends_on: 98 | pinot-controller: 99 | condition: service_healthy 100 | ports: 101 | - "8099:8099" 102 | volumes: 103 | - pinot_broker_data:/opt/pinot/data 104 | - ./configs:/opt/pinot/configs 105 | environment: 106 | LOG4J_CONSOLE_LEVEL: DEBUG 107 | JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log" 108 | -------------------------------------------------------------------------------- /ksqlDB/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | services: 7 | 8 | ## you can use ksql installed locally, just keep in mind it could be a different version 9 | ## as confluent installs ksql versioned as as part of confluent's versioning, not ksqldb's versioning 10 | ## ksql-cli: 11 | ## image: confluentinc/ksqldb-cli:${VERSION} 12 | ## hostname: ksql-cli 13 | ## container_name: ksql-cli 14 | ## depends_on: 15 | ## - ksql-server 16 | ## entrypoint: /bin/sh 17 | ## tty: true 18 | 19 | ksql-server: 20 | image: confluentinc/cp-ksqldb-server:${VERSION} 21 | hostname: ksql-server 22 | container_name: ksql-server 23 | healthcheck: 24 | test: curl -f http://localhost:8088/healthcheck || exit 1 25 | interval: 5s 26 | timeout: 5s 27 | retries: 20 28 | start_period: 20s 29 | ports: 30 | - '8088:8088' 31 | volumes: 32 | - ./jmx_prometheus:/opt/jmx_prometheus 33 | - ./ext:/etc/ksqldb/ext 34 | environment: 35 | KSQL_HOST_NAME: ksql-server 36 | KSQL_LISTENERS: "http://0.0.0.0:8088" 37 | KSQL_APPLICATION_ID: "ksql-server" 38 | KSQL_UDFS_ENABLED: "true" 39 | KSQL_KSQL_EXTENSION_DIR: "/etc/ksqldb/ext" 40 | KSQL_KSQL_METRICS_RECORDING_LEVEL_CONFIG: TRACE 41 | #KSQL_BOOTSTRAP_SERVERS: "broker-1-lb:9094,broker-2-lb:9094,broker-3-lb:9094,broker-4-lb:9094" 42 | KSQL_BOOTSTRAP_SERVERS: "broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092" 43 | KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" 44 | KSQL_CACHE_MAX_BYTES_BUFFERING: 0 45 | KSQL_LOG4J_ROOT_LOGLEVEL: DEBUG 46 | KSQL_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" 47 | KSQL_LOG4J_PROCESSING_LOG_BROKERLIST: broker-1:9092,broker-2:9092,broker-3:9092 48 | #KSQL_LOG4J_PROCESSING_LOG_TOPIC: 49 | #KSQL_KSQL_LOGGING_PROCESSING_TOPIC_NAME: 50 | KSQL_TOOLS_LOG4J_LOGLEVEL: DEBUG 51 | KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" 52 | KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" 53 | KSQL_PRODUCER_INTERCEPTOR_CLASSES: 54 | KSQL_CONSUMER_INTERCEPTOR_CLASSES: 55 | KSQL_KSQL_FAIL_ON_DESERIALIZATION_ERROR: "true" 56 | KSQL_KSQL_STREAMS_COMMIT_INTERVAL_MS: 8000 57 | KSQL_KSQL_STREAMS_PROCESSING_GUARANTEE: at_least_once 58 | KSQL_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/streams-config.yml 59 | 60 | # ksql-server-2: 61 | # image: confluentinc/cp-ksqldb-server:${VERSION} 62 | # hostname: ksql-server-2 63 | # container_name: ksql-server-2 64 | # healthcheck: 65 | # test: curl -f http://localhost:8088/healthcheck || exit 1 66 | # interval: 5s 67 | # timeout: 5s 68 | # retries: 5 69 | # start_period: 60s 70 | # volumes: 71 | # - ./jmx_prometheus:/opt/jmx_prometheus 72 | # - ./ext:/etc/ksqldb/ext 73 | # environment: 74 | # KSQL_HOST_NAME: ksql-server 75 | # KSQL_LISTENERS: "http://0.0.0.0:8088" 76 | # KSQL_APPLICATION_ID: "ksql-server" 77 | # KSQL_UDFS_ENABLED: "true" 78 | # KSQL_KSQL_EXTENSION_DIR: "/etc/ksqldb/ext" 79 | # KSQL_KSQL_METRICS_RECORDING_LEVEL_CONFIG: TRACE 80 | # #KSQL_BOOTSTRAP_SERVERS: "broker-1-lb:9094,broker-2-lb:9094,broker-3-lb:9094,broker-4-lb:9094" 81 | # KSQL_BOOTSTRAP_SERVERS: "broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092" 82 | # KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" 83 | # KSQL_CACHE_MAX_BYTES_BUFFERING: 0 84 | # KSQL_LOG4J_ROOT_LOGLEVEL: DEBUG 85 | # KSQL_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" 86 | # KSQL_LOG4J_PROCESSING_LOG_BROKERLIST: broker-1:9092,broker-2:9092,broker-3:9092 87 | # #KSQL_LOG4J_PROCESSING_LOG_TOPIC: 88 | # #KSQL_KSQL_LOGGING_PROCESSING_TOPIC_NAME: 89 | # KSQL_TOOLS_LOG4J_LOGLEVEL: DEBUG 90 | # KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" 91 | # KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" 92 | # KSQL_PRODUCER_INTERCEPTOR_CLASSES: 93 | # KSQL_CONSUMER_INTERCEPTOR_CLASSES: 94 | # KSQL_KSQL_FAIL_ON_DESERIALIZATION_ERROR: "true" 95 | # KSQL_KSQL_STREAMS_COMMIT_INTERVAL_MS: 8000 96 | # KSQL_KSQL_STREAMS_PROCESSING_GUARANTEE: at_least_once 97 | # KSQL_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/streams-config.yml 98 | -------------------------------------------------------------------------------- /connect/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | c1_kafka_data: {} 9 | c1_kafka_secrets: {} 10 | c2_kafka_data: {} 11 | c2_kafka_secrets: {} 12 | 13 | services: 14 | 15 | connect-1: 16 | image: confluentinc/cp-kafka-connect:${VERSION} 17 | hostname: connect-1 18 | container_name: connect-1 19 | ports: 20 | - "8083:8083" 21 | - "18083:8083" 22 | - "17072:7071" 23 | healthcheck: 24 | test: curl --fail http://localhost:8083/connector-plugins || exit 1 25 | interval: 5s 26 | timeout: 5s 27 | retries: 20 28 | start_period: 15s 29 | volumes: 30 | - c1_kafka_data:/var/lib/kafka/data 31 | - c1_kafka_secrets:/etc/kafka/secrets 32 | - ./jmx_prometheus:/opt/jmx_prometheus 33 | - ./jars:/etc/kafka-connect/jars 34 | - ./data:/data 35 | - ./secrets:/etc/kafka-connect/secrets 36 | environment: 37 | #CONNECT_BOOTSTRAP_SERVERS: 'broker-1-lb:9094,broker-2-lb:9094,broker-3-lb:9094,broker-4-lb:9094' 38 | CONNECT_BOOTSTRAP_SERVERS: 'broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092' 39 | CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: All 40 | CONNECT_REST_PORT: 8083 41 | CONNECT_GROUP_ID: 'connect-cluster' 42 | CONNECT_CONFIG_STORAGE_TOPIC: 'connect-cluster-config' 43 | CONNECT_OFFSET_STORAGE_TOPIC: 'connect-cluster-offsets' 44 | CONNECT_STATUS_STORAGE_TOPIC: 'connect-cluster-status' 45 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3 46 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3 47 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3 48 | CONNECT_CONFIG_PROVIDERS: file 49 | CONNECT_CONFIG_PROVIDERS_FILE_CLASS: 'org.apache.kafka.common.config.provider.FileConfigProvider' 50 | CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' 51 | CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' 52 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' 53 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' 54 | CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' 55 | CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' 56 | CONNECT_REST_ADVERTISED_HOST_NAME: 'connect-1' 57 | CONNECT_PLUGIN_PATH: /etc/kafka-connect/jars 58 | CONNECT_LOG4J_ROOT_LOGLEVEL: INFO 59 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/connect-config.yml 60 | 61 | connect-2: 62 | image: confluentinc/cp-kafka-connect:${VERSION} 63 | hostname: connect-2 64 | container_name: connect-2 65 | ports: 66 | - "28083:8083" 67 | healthcheck: 68 | test: curl --fail http://localhost:8083/connector-plugins || exit 1 69 | interval: 5s 70 | timeout: 5s 71 | retries: 20 72 | start_period: 15s 73 | volumes: 74 | - c2_kafka_data:/var/lib/kafka/data 75 | - c2_kafka_secrets:/etc/kafka/secrets 76 | - ./jmx_prometheus:/opt/jmx_prometheus 77 | - ./jars:/etc/kafka-connect/jars 78 | - ./data:/data 79 | - ./secrets:/etc/kafka-connect/secrets 80 | environment: 81 | #CONNECT_BOOTSTRAP_SERVERS: 'broker-1-lb:9094,broker-2-lb:9094,broker-3-lb:9094,broker-4-lb:9094' 82 | CONNECT_BOOTSTRAP_SERVERS: 'broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092' 83 | CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: All 84 | CONNECT_REST_PORT: 8083 85 | CONNECT_GROUP_ID: 'connect-cluster' 86 | CONNECT_CONFIG_STORAGE_TOPIC: 'connect-cluster-config' 87 | CONNECT_OFFSET_STORAGE_TOPIC: 'connect-cluster-offsets' 88 | CONNECT_STATUS_STORAGE_TOPIC: 'connect-cluster-status' 89 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3 90 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3 91 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3 92 | CONNECT_CONFIG_PROVIDERS: file 93 | CONNECT_CONFIG_PROVIDERS_FILE_CLASS: 'org.apache.kafka.common.config.provider.FileConfigProvider' 94 | CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' 95 | CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' 96 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' 97 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' 98 | CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' 99 | CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' 100 | CONNECT_REST_ADVERTISED_HOST_NAME: 'connect-2' 101 | CONNECT_PLUGIN_PATH: /etc/kafka-connect/jars 102 | CONNECT_LOG4J_ROOT_LOGLEVEL: INFO 103 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/connect-config.yml 104 | -------------------------------------------------------------------------------- /bin/connect: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | if ! [ -x "$(command -v jq)" ]; then 5 | echo "" 6 | echo "jq is not found, please install and make it available on your path, https://stedolan.github.io/jq" 7 | echo "" 8 | exit 9 | fi 10 | 11 | CONNECT_URL="http://localhost:18083" 12 | 13 | ACCEPT="Accept:application/json" 14 | CONTENT_TYPE="Content-Type:application/json" 15 | 16 | if [ $# -eq 1 ]; then 17 | case "$1" in 18 | available) 19 | curl -k -s -H "${ACCEPT}" -H "${CONTENT_TYPE}" -X GET ${CONNECT_URL}/connector-plugins 2>&1 >/dev/null 20 | [ $? -ne 0 ] && exit 1 21 | exit 0 22 | ;; 23 | plugins) 24 | curl -k -s -H "${ACCEPT}" -H "${CONTENT_TYPE}" -X GET ${CONNECT_URL}/connector-plugins | jq 'sort_by(.)' 25 | exit 26 | ;; 27 | connectors) 28 | curl -k -s -H "${ACCEPT}" -H "${CONTENT_TYPE}" -X GET ${CONNECT_URL}/connectors | jq 'sort_by(.)' 29 | exit 30 | ;; 31 | esac 32 | fi 33 | 34 | 35 | if [ $# -lt 2 ]; then 36 | echo "" 37 | echo "Usage: " 38 | echo " $0 {available|plugins|connectors}" 39 | echo " $0 {create|pause|resume|delete|status|validate} name" 40 | echo "" 41 | exit 42 | fi 43 | 44 | OPERATION=$1 45 | shift 46 | 47 | NAME=$1 48 | shift 49 | 50 | 51 | if [[ $NAME == *.json ]]; then 52 | NAME=${NAME%.*} 53 | fi 54 | 55 | CONFIG=${NAME}.json 56 | 57 | if [[ ( $OPERATION == "create" || $OPERATION == "update" ) && ! -f ${CONFIG} ]]; then 58 | echo "Usage: $0 {get|create|update|pause|resume|delete|status|validate} name" 59 | echo "" 60 | echo " $0 ${command} requires a valid configuration, ./${NAME}.json does not exist" 61 | echo "" 62 | # exit 63 | fi 64 | 65 | cat ${CONFIG} | jq > /dev/null 66 | 67 | if [ $? -ne 0 ]; then 68 | echo"" 69 | exit 70 | fi 71 | 72 | 73 | 74 | NAME=$(basename $NAME) 75 | 76 | echo "" 77 | echo "name : $NAME" 78 | echo "config file : $CONFIG" 79 | echo "" 80 | 81 | if [ -f ${CONFIG} ]; then 82 | 83 | PUT_DATA=$(cat < ${CONFIG}) 84 | 85 | POST_DATA=$(cat <([^:]+):" 33 | name: "kafka_schema_registry_jetty_metrics_$1" 34 | # "kafka.schema.registry:type=jersey-metrics" 35 | - pattern: "kafka.schema.registry([^:]+):" 36 | name: "kafka_schema_registry_jersey_metrics_$1" 37 | # "kafka.schema.registry:type=app-info,id=*" 38 | - pattern: "kafka.schema.registry<>(.+): (.+)" 39 | name: "kafka_schema_registry_app_info" 40 | value: 1 41 | labels: 42 | client-id: "$1" 43 | $2: "$3" 44 | type: UNTYPED 45 | # "kafka.schema.registry:type=registered-count" 46 | - pattern: "kafka.schema.registry([^:]+):" 47 | name: "kafka_schema_registry_registered_count" 48 | 49 | # "kafka.schema.registry:type=json-schemas-created" 50 | # "kafka.schema.registry:type=json-schemas-deleted" 51 | # "kafka.schema.registry:type=protobuf-schemas-created" 52 | # "kafka.schema.registry:type=protobuf-schemas-deleted" 53 | # "kafka.schema.registry:type=avro-schemas-created" 54 | # "kafka.schema.registry:type=avro-schemas-deleted" 55 | - pattern: "kafka.schema.registry([^:]+):" 56 | name: "kafka_schema_registry_schemas_$2" 57 | labels: 58 | schema_type: $1 59 | # kafka.schema.registry:type=kafka.schema.registry-metrics,client-id=* 60 | # kafka.schema.registry:type=kafka.schema.registry-coordinator-metrics,client-id=* 61 | - pattern: "kafka.schema.registry<>([^:]+):" 62 | name: "kafka_schema_registry_$1_$3" 63 | labels: 64 | client_id: $2 65 | # "kafka.consumer:type=app-info,client-id=*" 66 | # "kafka.producer:type=app-info,client-id=*" 67 | - pattern: "kafka.(.+)<>(.+): (.+)" 68 | value: 1 69 | name: kafka_$1_app_info 70 | labels: 71 | client_type: $1 72 | client_id: $2 73 | $3: $4 74 | type: UNTYPED 75 | # "kafka.consumer:type=consumer-metrics,client-id=*, protocol=*, cipher=*" 76 | # "kafka.consumer:type=type=consumer-fetch-manager-metrics,client-id=*, topic=*, partition=*" 77 | # "kafka.producer:type=producer-metrics,client-id=*, protocol=*, cipher=*" 78 | - pattern: "kafka.(.+)<>(.+):" 79 | name: kafka_$1_$2_$9 80 | type: GAUGE 81 | labels: 82 | client_type: $1 83 | $3: "$4" 84 | $5: "$6" 85 | $7: "$8" 86 | # "kafka.consumer:type=consumer-node-metrics,client-id=*, node-id=*" 87 | # "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*, topic=*" 88 | # "kafka.producer:type=producer-node-metrics,client-id=*, node-id=*" 89 | # "kafka.producer:type=producer-topic-metrics,client-id=*, topic=*" 90 | - pattern: "kafka.(.+)<>(.+):" 91 | name: kafka_$1_$2_$7 92 | type: GAUGE 93 | labels: 94 | client_type: $1 95 | $3: "$4" 96 | $5: "$6" 97 | # "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*" 98 | # "kafka.consumer:type=consumer-metrics,client-id=*" 99 | # "kafka.producer:type=producer-metrics,client-id=*" 100 | - pattern: "kafka.(.+)<>(.+):" 101 | name: kafka_$1_$2_$5 102 | type: GAUGE 103 | labels: 104 | client_type: $1 105 | $3: "$4" 106 | - pattern: "kafka.(.+)<>(.+):" 107 | name: kafka_$1_$2_$3 108 | labels: 109 | client_type: $1 -------------------------------------------------------------------------------- /kafka-raft/jmx_prometheus/sr-config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | lowercaseOutputName: true 3 | lowercaseOutputLabelNames: true 4 | whitelistObjectNames: 5 | - kafka.schema.registry:type=jetty-metrics 6 | - kafka.schema.registry:type=jersey-metrics 7 | - kafka.schema.registry:type=app-info,id=* 8 | - kafka.schema.registry:type=registered-count 9 | - kafka.schema.registry:type=json-schema* 10 | - kafka.schema.registry:type=protobuf-schemas* 11 | - kafka.schema.registry:type=avro-schemas* 12 | - kafka.schema.registry:type=kafka.schema.registry-metrics,client-id=* 13 | - kafka.schema.registry:type=kafka.schema.registry-coordinator-metrics,client-id=* 14 | # The two lines below are used to pull the Kafka Client Producer & consumer metrics from SR Client. 15 | # If you care about Producer/Consumer metrics for SR, please uncomment 2 lines below. 16 | # Please note that this increases the scrape duration to about 1 second as it needs to parse a lot of data. 17 | - "kafka.consumer:*" 18 | - "kafka.producer:*" 19 | blacklistObjectNames: 20 | # This will ignore the admin client metrics from SR server and will blacklist certain metrics 21 | # that do not make sense for ingestion. 22 | - kafka.producer:type=app-info,client-id=* 23 | - kafka.consumer:type=app-info,client-id=* 24 | - "kafka.admin.client:*" 25 | - "kafka.consumer:type=*,id=*" 26 | - "kafka.producer:type=*,id=*" 27 | # - "kafka.producer:client-id=confluent.monitoring*,*" 28 | # - "kafka.producer:client-id=confluent-license*,*" 29 | - "kafka.*:type=kafka-metrics-count,*" 30 | rules: 31 | # "kafka.schema.registry:type=jetty-metrics" 32 | - pattern: "kafka.schema.registry([^:]+):" 33 | name: "kafka_schema_registry_jetty_metrics_$1" 34 | # "kafka.schema.registry:type=jersey-metrics" 35 | - pattern: "kafka.schema.registry([^:]+):" 36 | name: "kafka_schema_registry_jersey_metrics_$1" 37 | # "kafka.schema.registry:type=app-info,id=*" 38 | - pattern: "kafka.schema.registry<>(.+): (.+)" 39 | name: "kafka_schema_registry_app_info" 40 | value: 1 41 | labels: 42 | client-id: "$1" 43 | $2: "$3" 44 | type: UNTYPED 45 | # "kafka.schema.registry:type=registered-count" 46 | - pattern: "kafka.schema.registry([^:]+):" 47 | name: "kafka_schema_registry_registered_count" 48 | 49 | # "kafka.schema.registry:type=json-schemas-created" 50 | # "kafka.schema.registry:type=json-schemas-deleted" 51 | # "kafka.schema.registry:type=protobuf-schemas-created" 52 | # "kafka.schema.registry:type=protobuf-schemas-deleted" 53 | # "kafka.schema.registry:type=avro-schemas-created" 54 | # "kafka.schema.registry:type=avro-schemas-deleted" 55 | - pattern: "kafka.schema.registry([^:]+):" 56 | name: "kafka_schema_registry_schemas_$2" 57 | labels: 58 | schema_type: $1 59 | # kafka.schema.registry:type=kafka.schema.registry-metrics,client-id=* 60 | # kafka.schema.registry:type=kafka.schema.registry-coordinator-metrics,client-id=* 61 | - pattern: "kafka.schema.registry<>([^:]+):" 62 | name: "kafka_schema_registry_$1_$3" 63 | labels: 64 | client_id: $2 65 | # "kafka.consumer:type=app-info,client-id=*" 66 | # "kafka.producer:type=app-info,client-id=*" 67 | - pattern: "kafka.(.+)<>(.+): (.+)" 68 | value: 1 69 | name: kafka_$1_app_info 70 | labels: 71 | client_type: $1 72 | client_id: $2 73 | $3: $4 74 | type: UNTYPED 75 | # "kafka.consumer:type=consumer-metrics,client-id=*, protocol=*, cipher=*" 76 | # "kafka.consumer:type=type=consumer-fetch-manager-metrics,client-id=*, topic=*, partition=*" 77 | # "kafka.producer:type=producer-metrics,client-id=*, protocol=*, cipher=*" 78 | - pattern: "kafka.(.+)<>(.+):" 79 | name: kafka_$1_$2_$9 80 | type: GAUGE 81 | labels: 82 | client_type: $1 83 | $3: "$4" 84 | $5: "$6" 85 | $7: "$8" 86 | # "kafka.consumer:type=consumer-node-metrics,client-id=*, node-id=*" 87 | # "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*, topic=*" 88 | # "kafka.producer:type=producer-node-metrics,client-id=*, node-id=*" 89 | # "kafka.producer:type=producer-topic-metrics,client-id=*, topic=*" 90 | - pattern: "kafka.(.+)<>(.+):" 91 | name: kafka_$1_$2_$7 92 | type: GAUGE 93 | labels: 94 | client_type: $1 95 | $3: "$4" 96 | $5: "$6" 97 | # "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*" 98 | # "kafka.consumer:type=consumer-metrics,client-id=*" 99 | # "kafka.producer:type=producer-metrics,client-id=*" 100 | - pattern: "kafka.(.+)<>(.+):" 101 | name: kafka_$1_$2_$5 102 | type: GAUGE 103 | labels: 104 | client_type: $1 105 | $3: "$4" 106 | - pattern: "kafka.(.+)<>(.+):" 107 | name: kafka_$1_$2_$3 108 | labels: 109 | client_type: $1 -------------------------------------------------------------------------------- /kafka-zk/jmx_prometheus/sr-config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | lowercaseOutputName: true 3 | lowercaseOutputLabelNames: true 4 | whitelistObjectNames: 5 | - java.lang:type=Runtime 6 | - kafka.schema.registry:type=jetty-metrics 7 | - kafka.schema.registry:type=jersey-metrics 8 | - kafka.schema.registry:type=app-info,id=* 9 | - kafka.schema.registry:type=registered-count 10 | - kafka.schema.registry:type=json-schema* 11 | - kafka.schema.registry:type=protobuf-schemas* 12 | - kafka.schema.registry:type=avro-schemas* 13 | - kafka.schema.registry:type=kafka.schema.registry-metrics,client-id=* 14 | - kafka.schema.registry:type=kafka.schema.registry-coordinator-metrics,client-id=* 15 | # The two lines below are used to pull the Kafka Client Producer & consumer metrics from SR Client. 16 | # If you care about Producer/Consumer metrics for SR, please uncomment 2 lines below. 17 | # Please note that this increases the scrape duration to about 1 second as it needs to parse a lot of data. 18 | - "kafka.consumer:*" 19 | - "kafka.producer:*" 20 | blacklistObjectNames: 21 | # This will ignore the admin client metrics from SR server and will blacklist certain metrics 22 | # that do not make sense for ingestion. 23 | - kafka.producer:type=app-info,client-id=* 24 | - kafka.consumer:type=app-info,client-id=* 25 | - "kafka.admin.client:*" 26 | - "kafka.consumer:type=*,id=*" 27 | - "kafka.producer:type=*,id=*" 28 | # - "kafka.producer:client-id=confluent.monitoring*,*" 29 | # - "kafka.producer:client-id=confluent-license*,*" 30 | - "kafka.*:type=kafka-metrics-count,*" 31 | rules: 32 | - pattern: java.lang 33 | # "kafka.schema.registry:type=jetty-metrics" 34 | - pattern: "kafka.schema.registry([^:]+):" 35 | name: "kafka_schema_registry_jetty_metrics_$1" 36 | # "kafka.schema.registry:type=jersey-metrics" 37 | - pattern: "kafka.schema.registry([^:]+):" 38 | name: "kafka_schema_registry_jersey_metrics_$1" 39 | # "kafka.schema.registry:type=app-info,id=*" 40 | - pattern: "kafka.schema.registry<>(.+): (.+)" 41 | name: "kafka_schema_registry_app_info" 42 | value: 1 43 | labels: 44 | client-id: "$1" 45 | $2: "$3" 46 | type: UNTYPED 47 | # "kafka.schema.registry:type=registered-count" 48 | - pattern: "kafka.schema.registry([^:]+):" 49 | name: "kafka_schema_registry_registered_count" 50 | 51 | # "kafka.schema.registry:type=json-schemas-created" 52 | # "kafka.schema.registry:type=json-schemas-deleted" 53 | # "kafka.schema.registry:type=protobuf-schemas-created" 54 | # "kafka.schema.registry:type=protobuf-schemas-deleted" 55 | # "kafka.schema.registry:type=avro-schemas-created" 56 | # "kafka.schema.registry:type=avro-schemas-deleted" 57 | - pattern: "kafka.schema.registry([^:]+):" 58 | name: "kafka_schema_registry_schemas_$2" 59 | labels: 60 | schema_type: $1 61 | # kafka.schema.registry:type=kafka.schema.registry-metrics,client-id=* 62 | # kafka.schema.registry:type=kafka.schema.registry-coordinator-metrics,client-id=* 63 | - pattern: "kafka.schema.registry<>([^:]+):" 64 | name: "kafka_schema_registry_$1_$3" 65 | labels: 66 | client_id: $2 67 | # "kafka.consumer:type=app-info,client-id=*" 68 | # "kafka.producer:type=app-info,client-id=*" 69 | - pattern: "kafka.(.+)<>(.+): (.+)" 70 | value: 1 71 | name: kafka_$1_app_info 72 | labels: 73 | client_type: $1 74 | client_id: $2 75 | $3: $4 76 | type: UNTYPED 77 | # "kafka.consumer:type=consumer-metrics,client-id=*, protocol=*, cipher=*" 78 | # "kafka.consumer:type=type=consumer-fetch-manager-metrics,client-id=*, topic=*, partition=*" 79 | # "kafka.producer:type=producer-metrics,client-id=*, protocol=*, cipher=*" 80 | - pattern: "kafka.(.+)<>(.+):" 81 | name: kafka_$1_$2_$9 82 | type: GAUGE 83 | labels: 84 | client_type: $1 85 | $3: "$4" 86 | $5: "$6" 87 | $7: "$8" 88 | # "kafka.consumer:type=consumer-node-metrics,client-id=*, node-id=*" 89 | # "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*, topic=*" 90 | # "kafka.producer:type=producer-node-metrics,client-id=*, node-id=*" 91 | # "kafka.producer:type=producer-topic-metrics,client-id=*, topic=*" 92 | - pattern: "kafka.(.+)<>(.+):" 93 | name: kafka_$1_$2_$7 94 | type: GAUGE 95 | labels: 96 | client_type: $1 97 | $3: "$4" 98 | $5: "$6" 99 | # "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*" 100 | # "kafka.consumer:type=consumer-metrics,client-id=*" 101 | # "kafka.producer:type=producer-metrics,client-id=*" 102 | - pattern: "kafka.(.+)<>(.+):" 103 | name: kafka_$1_$2_$5 104 | type: GAUGE 105 | labels: 106 | client_type: $1 107 | $3: "$4" 108 | - pattern: "kafka.(.+)<>(.+):" 109 | name: kafka_$1_$2_$3 110 | labels: 111 | client_type: $1 -------------------------------------------------------------------------------- /druid/environment: -------------------------------------------------------------------------------- 1 | # Java tuning 2 | DRUID_XMS=1g 3 | DRUID_XMX=1g 4 | DRUID_MAXNEWSIZE=250m 5 | DRUID_NEWSIZE=250m 6 | DRUID_MAXDIRECTMEMORYSIZE=6172m 7 | 8 | druid_emitter_logging_logLevel=debug 9 | 10 | #druid_extensions_loadList="[\"druid-kafka-indexing-service\", \"druid-histogram\", \"druid-datasketches\", \"druid-lookups-cached-global\", \"postgresql-metadata-storage\", \"kafka-emitter\", \"druid-avro-extensions\", \"druid-parquet-extensions\", \"druid-protobuf-extensions\"]" 11 | druid_extensions_loadList="[\"druid-kafka-indexing-service\", \"druid-histogram\", \"druid-datasketches\", \"druid-lookups-cached-global\", \"postgresql-metadata-storage\", \"kafka-emitter\", \"druid-avro-extensions\", \"druid-parquet-extensions\", \"druid-protobuf-extensions\", \"druid-s3-extensions\"]" 12 | 13 | 14 | 15 | #druid_extensions_loadList="[\"druid-kafka-indexing-service\", \"druid-histogram\", \"druid-datasketches\", \"druid-lookups-cached-global\", \"postgresql-metadata-storage\", \"kafka-emitter\", \"druid-avro-extensions\", \"druid-parquet-extensions\", \"prometheus-emitter\"]" 16 | #druid_extensions_loadList=[\"druid-kafka-indexing-service\", \"druid-histogram\", \"druid-datasketches\", \"druid-lookups-cached-global\", \"postgresql-metadata-storage\", \"kafka-emitter\"] 17 | #druid_extensions_loadList=[\"druid-kafka-indexing-service\", \"druid-histogram\", \"druid-datasketches\", \"druid-lookups-cached-global\", \"postgresql-metadata-storage\", \"druid-google-extensions\"] 18 | #druid_extensions_loadList=[\"druid-kafka-indexing-service\", \"druid-histogram\", \"druid-datasketches\", \"druid-lookups-cached-global\", \"postgresql-metadata-storage\", \"druid-s3-extensions\", \"druid-google-extensions\"] 19 | 20 | 21 | 22 | druid_zk_service_host=druid-zookeeper 23 | 24 | druid_metadata_storage_host= 25 | druid_metadata_storage_type=postgresql 26 | druid_metadata_storage_connector_connectURI=jdbc:postgresql://druid-postgres:5432/druid 27 | druid_metadata_storage_connector_user=druid 28 | druid_metadata_storage_connector_password=FoolishPassword 29 | 30 | druid_coordinator_balancer_strategy=cachingCost 31 | 32 | druid_worker_capacity=17 33 | 34 | # if both are set, both get applied, and javaOpts will get applied first, and dockerized containers set this one, not the Array property. 35 | #druid_indexer_runner_javaOpts="" 36 | #druid_indexer_runner_javaOpts="-Xdebug -Xrunjdwp:transport=dt_socket,address=0.0.0.0:5001,suspend=y" 37 | 38 | druid_indexer_runner_javaOptsArray="[\"-server\", \"-Xmx512g\", \"-Xms256m\", \"-XX:MaxDirectMemorySize=3g\", \"-Duser.timezone=UTC\", \"-Dfile.encoding=UTF-8\", \"-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager\"]" 39 | 40 | #druid_indexer_runner_javaOptsArray="[\"-server\", \"-Xmx512g\", \"-Xms256m\", \"-XX:MaxDirectMemorySize=3g\", \"-Duser.timezone=UTC\", \"-Dfile.encoding=UTF-8\", \"-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager\", \"-Xdebug\", \"-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5001\" ]" 41 | 42 | druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456 43 | 44 | druid_storage_type=local 45 | druid_storage_storageDirectory=/opt/shared/segments 46 | druid_indexer_logs_type=file 47 | druid_indexer_logs_directory=/opt/shared/indexing-logs 48 | 49 | 50 | # 6 is too much for my local setup, ... 51 | druid_processing_numThreads=2 52 | druid_processing_numMergeBuffers=2 53 | 54 | # Druid Metrics Emitter - KAFKA 55 | druid_emitter=kafka 56 | druid_emitter_kafka_bootstrap_servers=broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092 57 | druid_emitter_kafka_metric_topic=druid-metric 58 | druid_emitter_kafka_alert_topic=druid-alert 59 | druid_emitter_kafka_producer_config="{\"max.block.ms\":10000}" 60 | 61 | # Druid Metrics Emitter - HTTP 62 | #druid_emitter=http 63 | #druid_emitter_http_recipientBaseUrl=http://druid-exporter:7071/druid 64 | 65 | # Druid Metrics Emitter - Kafka & HTTP 66 | #druid_emitter=composing 67 | #druid_emitter_composing_emitters="[\"http\", \"kafka\"]" 68 | 69 | # 3rd party 70 | #druid_emitter=prometheus 71 | #druid_emitter_prometheus_strategy=exporter 72 | #druid_emitter_prometheus_port=7071 73 | 74 | 75 | 76 | #druid.query.groupBy.maxOnDiskStorage=2 77 | #druid.query.groupBy.maxMergingDictionarySize=300000000 78 | druid_query_groupBy_maxOnDiskStorage=4 79 | druid_query_groupBy_maxMergingDictionarySize=300000000 80 | 81 | DRUID_LOG4J="" 82 | 83 | s3service_s3_endpoint=minio 84 | s3service_s3_endpoint_http_port=9000 85 | s3service_disable_dns_buckets=true 86 | s3service_https_only=false 87 | 88 | #druid_s3_accessKey=admin 89 | #druid_s3_secretKey=miniominio 90 | druid_s3_accessKey=druid 91 | druid_s3_secretKey=druid_secret 92 | druid_s3_protocol=http 93 | druid_s3_endpoint_url=http://minio:9000 94 | druid_s3_enablePathStyleAccess=true 95 | #druid_s3_forceGlobalBucketAccessEnabled=true 96 | -------------------------------------------------------------------------------- /monitoring/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | prometheus: {} 8 | grafana: {} 9 | kafka-ui: {} 10 | 11 | services: 12 | prometheus: 13 | hostname: prometheus 14 | image: prom/prometheus:v2.47.2 15 | container_name: prometheus 16 | privileged: true 17 | healthcheck: 18 | test: wget --spider http://localhost:9090/ || exit 1 19 | start_period: 5s 20 | interval: 5s 21 | timeout: 5s 22 | retries: 3 23 | ports: 24 | - "9090:9090" 25 | volumes: 26 | - prometheus:/prometheus 27 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 28 | - ./prometheus/targets.json:/etc/prometheus/targets.json 29 | 30 | grafana: 31 | hostname: grafana 32 | image: grafana/grafana:main 33 | container_name: grafana 34 | healthcheck: 35 | test: wget --spider http://localhost:3000/ || exit 1 36 | start_period: 5s 37 | interval: 5s 38 | timeout: 5s 39 | retries: 3 40 | depends_on: 41 | prometheus: 42 | condition: service_healthy 43 | ports: 44 | - "3000:3000" 45 | environment: 46 | GF_SECURITY_ADMIN_PASSWORD: grafana 47 | GF_AUTH_ANONYMOUS_ENABLED: "true" 48 | GF_INSTALL_PLUGINS: "grafana-piechart-panel, grafadruid-druid-datasource" 49 | volumes: 50 | - grafana:/var/lib/grafana 51 | - ./grafana/grafana.ini:/etc/grafana/grafana.ini 52 | - ./grafana/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml 53 | - ./grafana/druid.yml:/etc/grafana/provisioning/datasources/druid.yml 54 | - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml 55 | - ./grafana/dashboards:/var/lib/grafana/dashboards 56 | 57 | kafka-exporter: 58 | hostname: kafka-exporter 59 | image: danielqsj/kafka-exporter:latest 60 | container_name: kafka-exporter 61 | healthcheck: 62 | test: wget --spider http://localhost:9308/ || exit 1 63 | start_period: 10s 64 | interval: 5s 65 | timeout: 5s 66 | retries: 3 67 | ports: 68 | - "9308:9308" 69 | command: --kafka.server=broker-1:9092 --kafka.server=broker-2:9092 --kafka.server=broker-3:9092 70 | kafka-ui: 71 | image: ghcr.io/kafbat/kafka-ui:latest 72 | container_name: kafka-ui 73 | healthcheck: 74 | test: curl --fail http://localhost:8080/ || exit 1 75 | start_period: 10s 76 | interval: 5s 77 | timeout: 5s 78 | retries: 3 79 | volumes: 80 | - kafka-ui:/tmp 81 | ports: 82 | - "9101:8080" 83 | restart: always 84 | environment: 85 | - KAFKA_CLUSTERS_0_NAME=local 86 | - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=broker-1:9092,broker-2:9092,broker-3:9092 87 | - KAFKA_CLUSTERS_0_ZOOKEEPER=zookeeper:2181 88 | - KAFKA_CLUSTERS_0_SCHEMAREGISTRY=http://schema-registry:8081 89 | - KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME=connect 90 | - KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS=http://connect-1:8083 91 | # 92 | # kowl: 93 | # image: docker.redpanda.com/vectorized/console:v2.1.1 94 | # hostname: kowl 95 | # container_name: kowl 96 | # healthcheck: 97 | # test: wget --spider http://localhost:8080/ || exit 1 98 | # start_period: 10s 99 | # interval: 5s 100 | # timeout: 5s 101 | # retries: 3 102 | # volumes: 103 | # - ./kowl/config.yaml:/etc/kowl/config.yaml 104 | # ports: 105 | # - "9100:8080" 106 | # environment: 107 | # - KAFKA_BROKERS=broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092 108 | # 109 | # 110 | # kouncil: 111 | # image: consdata/kouncil:1.4 112 | # container_name: kouncil 113 | # volumes: 114 | # - ./kouncil/kouncil.yaml:/config/kouncil.yaml 115 | # ports: 116 | # - "9102:8080" 117 | # environment: 118 | # bootstrapServers: "broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092" 119 | # 120 | # akhq: 121 | # image: tchiotludo/akhq:latest 122 | # container_name: akhq 123 | # volumes: 124 | # - ./akhq/application.yml:/app/application.yml 125 | # ports: 126 | # - "9103:8080" 127 | # 128 | # kafdrop: 129 | # image: obsidiandynamics/kafdrop 130 | # container_name: kafdrop 131 | # ports: 132 | # - "9104:9000" 133 | # environment: 134 | # KAFKA_BROKERCONNECT: broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092 135 | # 136 | ## druid-exporter: 137 | ## image: druid-exporter:latest 138 | ## container_name: druid-exporter 139 | ## command: 140 | ## - "conf/example_druid_v_0_17_0.json" -t druid-metric -b broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092 -g "druid-exporter" 141 | ## ports: 142 | ## - "9999:8000" 143 | # 144 | ## cmak_zk: 145 | ## image: zookeeper:latest 146 | ## restart: always 147 | ## environment: 148 | ## ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 149 | ## cmak: 150 | ## image: cmak:latest #hlebalbau/kafka-manager:stable 151 | ## hostname: cmak 152 | ## container_name: cmak 153 | ## restart: always 154 | ## command: 155 | ## - "-Dcmak.zkhosts=cmak_zk:2181" 156 | ## ports: 157 | ## - "9005:9000" 158 | -------------------------------------------------------------------------------- /mysql5/sql/queries.sql: -------------------------------------------------------------------------------- 1 | SELECT SUM(LO_EXTENDEDPRICE*LO_DISCOUNT) AS REVENUE 2 | FROM SSB.LINEORDER, SSB.DWDATE 3 | WHERE LO_ORDERDATE = D_DATEKEY 4 | AND D_YEARMONTHNUM = 199401 5 | AND LO_DISCOUNT BETWEEN 4 AND 6 6 | AND LO_QUANTITY BETWEEN 26 AND 35; 7 | 8 | SELECT SUM(LO_EXTENDEDPRICE*LO_DISCOUNT) AS REVENUE 9 | FROM SSB.LINEORDER, SSB.DWDATE 10 | WHERE LO_ORDERDATE = D_DATEKEY 11 | AND D_YEAR = 1993 12 | AND LO_DISCOUNT BETWEEN 1 AND 3 13 | AND LO_QUANTITY < 25; 14 | 15 | SELECT SUM(LO_EXTENDEDPRICE*LO_DISCOUNT) AS REVENUE 16 | FROM SSB.LINEORDER, SSB.DWDATE 17 | WHERE LO_ORDERDATE = D_DATEKEY 18 | AND D_YEARMONTHNUM = 199401 19 | AND LO_DISCOUNT BETWEEN 4 AND 6 20 | AND LO_QUANTITY BETWEEN 26 AND 35; 21 | 22 | SELECT SUM(LO_EXTENDEDPRICE*LO_DISCOUNT) AS REVENUE 23 | FROM SSB.LINEORDER, SSB.DWDATE 24 | WHERE LO_ORDERDATE = D_DATEKEY 25 | AND D_WEEKNUMINYEAR = 6 26 | AND D_YEAR = 1994 27 | AND LO_DISCOUNT BETWEEN 5 AND 7 28 | AND LO_QUANTITY BETWEEN 26 AND 35; 29 | 30 | SELECT SUM(LO_REVENUE), D_YEAR, P_BRAND1 31 | FROM SSB.LINEORDER, SSB.DWDATE, SSB.PART, SSB.SUPPLIER 32 | WHERE LO_ORDERDATE = D_DATEKEY 33 | AND LO_PARTKEY = P_PARTKEY 34 | AND LO_SUPPKEY = S_SUPPKEY 35 | AND P_CATEGORY = 'MFGR#12' 36 | AND S_REGION = 'AMERICA' 37 | GROUP BY D_YEAR, P_BRAND1 38 | ORDER BY D_YEAR, P_BRAND1; 39 | 40 | SELECT SUM(LO_REVENUE), D_YEAR, P_BRAND1 41 | FROM SSB.LINEORDER, SSB.DWDATE, SSB.PART, SSB.SUPPLIER 42 | WHERE LO_ORDERDATE = D_DATEKEY 43 | AND LO_PARTKEY = P_PARTKEY 44 | AND LO_SUPPKEY = S_SUPPKEY 45 | AND P_BRAND1 BETWEEN 'MFGR#2221' AND 'MFGR#2228' 46 | AND S_REGION = 'ASIA' 47 | GROUP BY D_YEAR, P_BRAND1 48 | ORDER BY D_YEAR, P_BRAND1; 49 | 50 | SELECT SUM(LO_REVENUE), D_YEAR, P_BRAND1 51 | FROM SSB.LINEORDER, SSB.DWDATE, SSB.PART, SSB.SUPPLIER 52 | WHERE LO_ORDERDATE = D_DATEKEY 53 | AND LO_PARTKEY = P_PARTKEY 54 | AND LO_SUPPKEY = S_SUPPKEY 55 | AND P_BRAND1 = 'MFGR#2221' 56 | AND S_REGION = 'EUROPE' 57 | GROUP BY D_YEAR, P_BRAND1 58 | ORDER BY D_YEAR, P_BRAND1; 59 | 60 | SELECT C_NATION, S_NATION, D_YEAR, SUM(LO_REVENUE) AS REVENUE 61 | FROM SSB.CUSTOMER, SSB.LINEORDER, SSB.SUPPLIER, SSB.DWDATE 62 | WHERE LO_CUSTKEY = C_CUSTKEY 63 | AND LO_SUPPKEY = S_SUPPKEY 64 | AND LO_ORDERDATE = D_DATEKEY 65 | AND C_REGION = 'ASIA' AND S_REGION = 'ASIA' 66 | AND D_YEAR >= 1992 AND D_YEAR <= 1997 67 | GROUP BY C_NATION, S_NATION, D_YEAR 68 | ORDER BY D_YEAR ASC, REVENUE DESC; 69 | 70 | SELECT C_CITY, S_CITY, D_YEAR, SUM(LO_REVENUE) AS REVENUE 71 | FROM SSB.CUSTOMER, SSB.LINEORDER, SSB.SUPPLIER, SSB.DWDATE 72 | WHERE LO_CUSTKEY = C_CUSTKEY 73 | AND LO_SUPPKEY = S_SUPPKEY 74 | AND LO_ORDERDATE = D_DATEKEY 75 | AND C_NATION = 'UNITED STATES' 76 | AND S_NATION = 'UNITED STATES' 77 | AND D_YEAR >= 1992 AND D_YEAR <= 1997 78 | GROUP BY C_CITY, S_CITY, D_YEAR 79 | ORDER BY D_YEAR ASC, REVENUE DESC; 80 | 81 | SELECT C_CITY, S_CITY, D_YEAR, SUM(LO_REVENUE) AS REVENUE 82 | FROM SSB.CUSTOMER, SSB.LINEORDER, SSB.SUPPLIER, SSB.DWDATE 83 | WHERE LO_CUSTKEY = C_CUSTKEY 84 | AND LO_SUPPKEY = S_SUPPKEY 85 | AND LO_ORDERDATE = D_DATEKEY 86 | AND (C_CITY='UNITED KI1' OR C_CITY='UNITED KI5') 87 | AND (S_CITY='UNITED KI1' OR S_CITY='UNITED KI5') 88 | AND D_YEAR >= 1992 AND D_YEAR <= 1997 89 | GROUP BY C_CITY, S_CITY, D_YEAR 90 | ORDER BY D_YEAR ASC, REVENUE DESC; 91 | 92 | SELECT C_CITY, S_CITY, D_YEAR, SUM(LO_REVENUE) AS REVENUE 93 | FROM SSB.CUSTOMER, SSB.LINEORDER, SSB.SUPPLIER, SSB.DWDATE 94 | WHERE LO_CUSTKEY = C_CUSTKEY 95 | AND LO_SUPPKEY = S_SUPPKEY 96 | AND LO_ORDERDATE = D_DATEKEY 97 | AND (C_CITY='UNITED KI1' OR C_CITY='UNITED KI5') 98 | AND (S_CITY='UNITED KI1' OR S_CITY='UNITED KI5') 99 | AND D_YEARMONTH = 'DEC1997' 100 | GROUP BY C_CITY, S_CITY, D_YEAR 101 | ORDER BY D_YEAR ASC, REVENUE DESC; 102 | 103 | SELECT D_YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS PROFIT 104 | FROM SSB.DWDATE, SSB.CUSTOMER, SSB.SUPPLIER, SSB.PART, SSB.LINEORDER 105 | WHERE LO_CUSTKEY = C_CUSTKEY 106 | AND LO_SUPPKEY = S_SUPPKEY 107 | AND LO_PARTKEY = P_PARTKEY 108 | AND LO_ORDERDATE = D_DATEKEY 109 | AND C_REGION = 'AMERICA' 110 | AND S_REGION = 'AMERICA' 111 | AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') 112 | GROUP BY D_YEAR, C_NATION 113 | ORDER BY D_YEAR, C_NATION; 114 | 115 | SELECT D_YEAR, S_NATION, P_CATEGORY, SUM(LO_REVENUE - LO_SUPPLYCOST) AS PROFIT 116 | FROM SSB.DWDATE, SSB.CUSTOMER, SSB.SUPPLIER, SSB.PART, SSB.LINEORDER 117 | WHERE LO_CUSTKEY = C_CUSTKEY 118 | AND LO_SUPPKEY = S_SUPPKEY 119 | AND LO_PARTKEY = P_PARTKEY 120 | AND LO_ORDERDATE = D_DATEKEY 121 | AND C_REGION = 'AMERICA' 122 | AND S_REGION = 'AMERICA' 123 | AND (D_YEAR = 1997 OR D_YEAR = 1998) 124 | AND (P_MFGR = 'MFGR#1' 125 | OR P_MFGR = 'MFGR#2') 126 | GROUP BY D_YEAR, S_NATION, P_CATEGORY ORDER BY D_YEAR, S_NATION, P_CATEGORY; 127 | 128 | SELECT D_YEAR, S_CITY, P_BRAND1, SUM(LO_REVENUE - LO_SUPPLYCOST) AS PROFIT 129 | FROM SSB.DWDATE, SSB.CUSTOMER, SSB.SUPPLIER, SSB.PART, SSB.LINEORDER 130 | WHERE LO_CUSTKEY = C_CUSTKEY 131 | AND LO_SUPPKEY = S_SUPPKEY 132 | AND LO_PARTKEY = P_PARTKEY 133 | AND LO_ORDERDATE = D_DATEKEY 134 | AND C_REGION = 'AMERICA' 135 | AND S_NATION = 'UNITED STATES' 136 | AND (D_YEAR = 1997 OR D_YEAR = 1998) 137 | AND P_CATEGORY = 'MFGR#14' 138 | GROUP BY D_YEAR, S_CITY, P_BRAND1 ORDER BY D_YEAR, S_CITY, P_BRAND1; -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/home.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "target": { 12 | "limit": 100, 13 | "matchAny": false, 14 | "tags": [], 15 | "type": "dashboard" 16 | }, 17 | "type": "dashboard" 18 | } 19 | ] 20 | }, 21 | "editable": true, 22 | "fiscalYearStartMonth": 0, 23 | "graphTooltip": 0, 24 | "links": [], 25 | "liveNow": false, 26 | "panels": [ 27 | { 28 | "description": "", 29 | "gridPos": { 30 | "h": 9, 31 | "w": 12, 32 | "x": 0, 33 | "y": 0 34 | }, 35 | "id": 19, 36 | "options": { 37 | "maxItems": 100, 38 | "query": "", 39 | "showHeadings": false, 40 | "showRecentlyViewed": false, 41 | "showSearch": true, 42 | "showStarred": false, 43 | "tags": [ 44 | "kafka_cluster" 45 | ] 46 | }, 47 | "pluginVersion": "8.3.3", 48 | "targets": [ 49 | { 50 | "queryType": "randomWalk", 51 | "refId": "A" 52 | } 53 | ], 54 | "title": "Kafka Cluster", 55 | "type": "dashlist" 56 | }, 57 | { 58 | "description": "", 59 | "gridPos": { 60 | "h": 9, 61 | "w": 12, 62 | "x": 12, 63 | "y": 0 64 | }, 65 | "id": 18, 66 | "options": { 67 | "maxItems": 100, 68 | "query": "", 69 | "showHeadings": false, 70 | "showRecentlyViewed": false, 71 | "showSearch": true, 72 | "showStarred": false, 73 | "tags": [ 74 | "connect_cluster" 75 | ] 76 | }, 77 | "pluginVersion": "8.3.3", 78 | "targets": [ 79 | { 80 | "queryType": "randomWalk", 81 | "refId": "A" 82 | } 83 | ], 84 | "title": "Kafka Connect", 85 | "type": "dashlist" 86 | }, 87 | { 88 | "description": "", 89 | "gridPos": { 90 | "h": 4, 91 | "w": 12, 92 | "x": 0, 93 | "y": 9 94 | }, 95 | "id": 21, 96 | "options": { 97 | "maxItems": 100, 98 | "query": "", 99 | "showHeadings": false, 100 | "showRecentlyViewed": false, 101 | "showSearch": true, 102 | "showStarred": false, 103 | "tags": [ 104 | "schema-registry" 105 | ] 106 | }, 107 | "pluginVersion": "8.3.3", 108 | "targets": [ 109 | { 110 | "queryType": "randomWalk", 111 | "refId": "A" 112 | } 113 | ], 114 | "title": "Schema Registry", 115 | "type": "dashlist" 116 | }, 117 | { 118 | "description": "", 119 | "gridPos": { 120 | "h": 11, 121 | "w": 12, 122 | "x": 12, 123 | "y": 9 124 | }, 125 | "id": 22, 126 | "options": { 127 | "maxItems": 100, 128 | "query": "", 129 | "showHeadings": false, 130 | "showRecentlyViewed": false, 131 | "showSearch": true, 132 | "showStarred": false, 133 | "tags": [ 134 | "streams" 135 | ] 136 | }, 137 | "pluginVersion": "8.3.3", 138 | "targets": [ 139 | { 140 | "queryType": "randomWalk", 141 | "refId": "A" 142 | } 143 | ], 144 | "title": "Kafka Streams", 145 | "type": "dashlist" 146 | }, 147 | { 148 | "description": "", 149 | "gridPos": { 150 | "h": 4, 151 | "w": 12, 152 | "x": 0, 153 | "y": 13 154 | }, 155 | "id": 20, 156 | "options": { 157 | "maxItems": 100, 158 | "query": "", 159 | "showHeadings": false, 160 | "showRecentlyViewed": false, 161 | "showSearch": true, 162 | "showStarred": false, 163 | "tags": [ 164 | "jvm" 165 | ] 166 | }, 167 | "pluginVersion": "8.3.3", 168 | "targets": [ 169 | { 170 | "queryType": "randomWalk", 171 | "refId": "A" 172 | } 173 | ], 174 | "title": "JVM", 175 | "type": "dashlist" 176 | }, 177 | { 178 | "description": "", 179 | "gridPos": { 180 | "h": 6, 181 | "w": 12, 182 | "x": 0, 183 | "y": 13 184 | }, 185 | "id": 23, 186 | "options": { 187 | "maxItems": 100, 188 | "query": "", 189 | "showHeadings": false, 190 | "showRecentlyViewed": false, 191 | "showSearch": true, 192 | "showStarred": false, 193 | "tags": [ 194 | "flink" 195 | ] 196 | }, 197 | "pluginVersion": "8.3.3", 198 | "targets": [ 199 | { 200 | "queryType": "randomWalk", 201 | "refId": "A" 202 | } 203 | ], 204 | "title": "Flink", 205 | "type": "dashlist" 206 | } 207 | ], 208 | "refresh": "", 209 | "schemaVersion": 34, 210 | "style": "dark", 211 | "tags": [ 212 | "home" 213 | ], 214 | "templating": { 215 | "list": [] 216 | }, 217 | "time": { 218 | "from": "now-1m", 219 | "to": "now" 220 | }, 221 | "timepicker": { 222 | "hidden": true, 223 | "refresh_intervals": [], 224 | "time_options": [] 225 | }, 226 | "timezone": "", 227 | "title": "Home", 228 | "uid": "home", 229 | "version": 1, 230 | "weekStart": "" 231 | } -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/druid/druid-ingestion.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "target": { 12 | "limit": 100, 13 | "matchAny": false, 14 | "tags": [], 15 | "type": "dashboard" 16 | }, 17 | "type": "dashboard" 18 | } 19 | ] 20 | }, 21 | "description": "", 22 | "editable": true, 23 | "fiscalYearStartMonth": 0, 24 | "gnetId": 12155, 25 | "graphTooltip": 0, 26 | "links": [], 27 | "liveNow": false, 28 | "panels": [ 29 | { 30 | "datasource": "prometheus", 31 | "fieldConfig": { 32 | "defaults": { 33 | "color": { 34 | "mode": "palette-classic" 35 | }, 36 | "custom": { 37 | "axisLabel": "", 38 | "axisPlacement": "auto", 39 | "barAlignment": 0, 40 | "drawStyle": "line", 41 | "fillOpacity": 0, 42 | "gradientMode": "none", 43 | "hideFrom": { 44 | "legend": false, 45 | "tooltip": false, 46 | "viz": false 47 | }, 48 | "lineInterpolation": "linear", 49 | "lineStyle": { 50 | "fill": "solid" 51 | }, 52 | "lineWidth": 1, 53 | "pointSize": 5, 54 | "scaleDistribution": { 55 | "type": "linear" 56 | }, 57 | "showPoints": "never", 58 | "spanNulls": true, 59 | "stacking": { 60 | "group": "A", 61 | "mode": "none" 62 | }, 63 | "thresholdsStyle": { 64 | "mode": "off" 65 | } 66 | }, 67 | "mappings": [], 68 | "thresholds": { 69 | "mode": "absolute", 70 | "steps": [ 71 | { 72 | "color": "green", 73 | "value": null 74 | }, 75 | { 76 | "color": "red", 77 | "value": 80 78 | } 79 | ] 80 | }, 81 | "unit": "short" 82 | }, 83 | "overrides": [] 84 | }, 85 | "gridPos": { 86 | "h": 12, 87 | "w": 24, 88 | "x": 0, 89 | "y": 0 90 | }, 91 | "id": 37, 92 | "options": { 93 | "legend": { 94 | "calcs": [ 95 | "mean", 96 | "max", 97 | "last" 98 | ], 99 | "displayMode": "table", 100 | "placement": "right" 101 | }, 102 | "tooltip": { 103 | "mode": "single" 104 | } 105 | }, 106 | "pluginVersion": "8.2.5", 107 | "targets": [ 108 | { 109 | "exemplar": true, 110 | "expr": "sum(increase(druid_emitted_metrics{metric_name=\"ingest-events-processed\"}[$__range])) by (host)", 111 | "interval": "", 112 | "legendFormat": "{{host}}", 113 | "refId": "A" 114 | } 115 | ], 116 | "timeFrom": null, 117 | "timeShift": null, 118 | "title": "Ingestion Events", 119 | "type": "timeseries" 120 | }, 121 | { 122 | "datasource": "prometheus", 123 | "fieldConfig": { 124 | "defaults": { 125 | "color": { 126 | "mode": "palette-classic" 127 | }, 128 | "custom": { 129 | "axisLabel": "", 130 | "axisPlacement": "auto", 131 | "barAlignment": 0, 132 | "drawStyle": "line", 133 | "fillOpacity": 0, 134 | "gradientMode": "none", 135 | "hideFrom": { 136 | "legend": false, 137 | "tooltip": false, 138 | "viz": false 139 | }, 140 | "lineInterpolation": "linear", 141 | "lineWidth": 1, 142 | "pointSize": 5, 143 | "scaleDistribution": { 144 | "type": "linear" 145 | }, 146 | "showPoints": "never", 147 | "spanNulls": true, 148 | "stacking": { 149 | "group": "A", 150 | "mode": "none" 151 | }, 152 | "thresholdsStyle": { 153 | "mode": "off" 154 | } 155 | }, 156 | "mappings": [], 157 | "thresholds": { 158 | "mode": "absolute", 159 | "steps": [ 160 | { 161 | "color": "green", 162 | "value": null 163 | }, 164 | { 165 | "color": "red", 166 | "value": 80 167 | } 168 | ] 169 | }, 170 | "unit": "short" 171 | }, 172 | "overrides": [] 173 | }, 174 | "gridPos": { 175 | "h": 11, 176 | "w": 24, 177 | "x": 0, 178 | "y": 12 179 | }, 180 | "id": 38, 181 | "options": { 182 | "legend": { 183 | "calcs": [ 184 | "mean", 185 | "max", 186 | "last" 187 | ], 188 | "displayMode": "table", 189 | "placement": "right" 190 | }, 191 | "tooltip": { 192 | "mode": "single" 193 | } 194 | }, 195 | "pluginVersion": "8.2.5", 196 | "targets": [ 197 | { 198 | "exemplar": true, 199 | "expr": "sum (increase(druid_emitted_metrics{metric_name=\"ingest-kafka-lag\"}[$__range])) by (datasource)", 200 | "interval": "", 201 | "legendFormat": "{{datasource}}", 202 | "refId": "A" 203 | } 204 | ], 205 | "timeFrom": null, 206 | "timeShift": null, 207 | "title": "Ingestion Lag", 208 | "type": "timeseries" 209 | } 210 | ], 211 | "refresh": false, 212 | "schemaVersion": 32, 213 | "style": "dark", 214 | "tags": [ 215 | "druid" 216 | ], 217 | "templating": { 218 | "list": [] 219 | }, 220 | "time": { 221 | "from": "now-1m", 222 | "to": "now" 223 | }, 224 | "timepicker": { 225 | "refresh_intervals": [ 226 | "10s", 227 | "30s", 228 | "1m", 229 | "5m", 230 | "15m", 231 | "30m", 232 | "1h", 233 | "2h", 234 | "1d" 235 | ] 236 | }, 237 | "timezone": "", 238 | "title": "Druid - Ingestion", 239 | "uid": "druid_ingestion", 240 | "version": 2 241 | } -------------------------------------------------------------------------------- /druid/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | networks: 3 | default: 4 | external: true 5 | name: dev-local 6 | 7 | volumes: 8 | postgres: {} 9 | zk: {} 10 | zk_datalog: {} 11 | zk_logs: {} 12 | coordinator_var: {} 13 | broker_var: {} 14 | historical_var: {} 15 | middlemanager_var: {} 16 | router_var: {} 17 | shared: {} 18 | 19 | services: 20 | druid-postgres: 21 | image: postgres:latest 22 | container_name: druid-postgres 23 | hostname: druid-postgres 24 | healthcheck: 25 | test: pg_isready -U druid 26 | interval: 10s 27 | timeout: 5s 28 | retries: 10 29 | ports: 30 | - "45432:5432" 31 | volumes: 32 | - postgres:/var/lib/postgresql/data 33 | environment: 34 | POSTGRES_PASSWORD: FoolishPassword 35 | POSTGRES_USER: druid 36 | POSTGRES_DB: druid 37 | 38 | # Need 3.5 or later for container nodes 39 | druid-zookeeper: 40 | image: zookeeper:3.5 41 | container_name: druid-zookeeper 42 | hostname: druid-zookeeper 43 | healthcheck: 44 | test: echo ruok | nc -w 2 localhost 2181 | grep imok 45 | start_period: 5s 46 | interval: 5s 47 | timeout: 5s 48 | retries: 3 49 | volumes: 50 | - zk:/data 51 | - zk_datalog:/datalog 52 | - zk_logs:/logs 53 | environment: 54 | ZOO_MY_ID: 1 55 | ZOO_4LW_COMMANDS_WHITELIST: "ruok,stat" 56 | # deploy: 57 | # resources: 58 | # limits: 59 | # cpus: '0.50' 60 | # memory: 50M 61 | # reservations: 62 | # memory: 20M 63 | 64 | 65 | druid-coordinator: 66 | image: apache/druid:${VERSION} 67 | container_name: druid-coordinator 68 | hostname: druid-coordinator 69 | healthcheck: 70 | test: wget --spider http://localhost:8081/status/health || exit 1 71 | interval: 5s 72 | timeout: 5s 73 | retries: 10 74 | start_period: 60s 75 | volumes: 76 | - coordinator_var:/opt/druid/var 77 | - shared:/opt/shared 78 | - ./jmx_prometheus:/opt/jmx_prometheus 79 | - ./staging:/opt/staging 80 | depends_on: 81 | - druid-zookeeper 82 | - druid-postgres 83 | ports: 84 | - "48081:8081" 85 | command: 86 | - coordinator 87 | env_file: 88 | - environment 89 | environment: 90 | JAVA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/druid-config.yml 91 | # -Djava.rmi.server.hostname=0.0.0.0 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.rmi.port=9999 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false 92 | 93 | druid-broker: 94 | image: apache/druid:${VERSION} 95 | container_name: druid-broker 96 | hostname: druid-broker 97 | healthcheck: 98 | test: wget --spider http://localhost:8082/status/health || exit 1 99 | interval: 5s 100 | timeout: 5s 101 | retries: 10 102 | start_period: 30s 103 | volumes: 104 | - broker_var:/opt/druid/var 105 | - ./jmx_prometheus:/opt/jmx_prometheus 106 | - ./staging:/opt/staging 107 | depends_on: 108 | - druid-zookeeper 109 | - druid-postgres 110 | - druid-coordinator 111 | ports: 112 | - "48082:8082" 113 | command: 114 | - broker 115 | env_file: 116 | - environment 117 | environment: 118 | JAVA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/druid-config.yml 119 | # deploy: 120 | # resources: 121 | # limits: 122 | # memory: 1.25G 123 | 124 | druid-historical: 125 | image: apache/druid:${VERSION} 126 | container_name: druid-historical 127 | hostname: druid-historical 128 | healthcheck: 129 | test: wget --spider http://localhost:8083/druid/historical/v1/readiness || exit 1 130 | interval: 5s 131 | timeout: 5s 132 | retries: 10 133 | start_period: 30s 134 | volumes: 135 | - historical_var:/opt/druid/var 136 | - shared:/opt/shared 137 | - ./jmx_prometheus:/opt/jmx_prometheus 138 | - ./staging:/opt/staging 139 | depends_on: 140 | - druid-zookeeper 141 | - druid-postgres 142 | - druid-coordinator 143 | ports: 144 | - "48083:8083" 145 | command: 146 | - historical 147 | env_file: 148 | - environment 149 | environment: 150 | JAVA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/druid-config.yml 151 | 152 | druid-middlemanager: 153 | image: apache/druid:${VERSION} 154 | container_name: druid-middlemanager 155 | hostname: druid-middlemanager 156 | # deploy: 157 | # replicas: 2 158 | healthcheck: 159 | test: wget --spider http://localhost:8091/status/health || exit 1 160 | interval: 5s 161 | timeout: 5s 162 | retries: 10 163 | start_period: 30s 164 | volumes: 165 | - middlemanager_var:/opt/druid/var 166 | - shared:/opt/shared 167 | - ./jmx_prometheus:/opt/jmx_prometheus 168 | - ./staging:/opt/staging:rwx 169 | depends_on: 170 | - druid-zookeeper 171 | - druid-postgres 172 | - druid-coordinator 173 | ports: 174 | - "48091:8091" 175 | - "5001:5001" 176 | command: 177 | - middleManager 178 | env_file: 179 | - environment 180 | environment: 181 | JAVA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/druid-config.yml 182 | 183 | druid-router: 184 | image: apache/druid:${VERSION} 185 | container_name: druid-router 186 | hostname: druid-router 187 | healthcheck: 188 | test: wget --spider http://localhost:8888/status/health || exit 1 189 | interval: 5s 190 | timeout: 5s 191 | retries: 10 192 | start_period: 30s 193 | volumes: 194 | - router_var:/opt/druid/var 195 | - ./jmx_prometheus:/opt/jmx_prometheus 196 | - ./staging:/opt/staging 197 | depends_on: 198 | - druid-zookeeper 199 | - druid-postgres 200 | - druid-coordinator 201 | ports: 202 | - "48888:8888" 203 | command: 204 | - router 205 | env_file: 206 | - environment 207 | environment: 208 | JAVA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/druid-config.yml 209 | 210 | # #this is not part of Apache Druid 211 | # druid-exporter: 212 | # image: quay.io/opstree/druid-exporter:v0.11 213 | # container_name: druid-exporter 214 | # environment: 215 | # DRUID_URL: "http://druid-router:8888" 216 | # PORT: "7071" 217 | # ports: 218 | # - "17073:7071" 219 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | * This project is to provide a complete ecosystem to do local Kafka development w/out the need to connect to external systems. 4 | 5 | * Each directory has its own `docker-compose.yml` and its own docker-compose project defined in the `.env` property file. 6 | 7 | * this makes it easy to start/stop only what you need. 8 | 9 | * the network is created as external to allow for the containers between compose files to communicate while also not 10 | requiring any specific docker compose to be running. 11 | 12 | * Even though you are running all of the Kafka components from docker containers, you will want the software installed 13 | locally so you can use the command line operations; personally, I install the confluent community edition installation 14 | locally in `/usr/local/confluent`, and add `/usr/local/confluent/bin` to my path. 15 | 16 | * Go to Confluent's [get-started](https://www.confluent.io/get-started), select the **here** link (javascript so I cannot deep link it) 17 | in the Download Confluent Platform section to get the download for community edition. 18 | 19 | * On A Mac M1 Laptop, Java 17 or later is needed. The `KSQL_JVM_PERFORMANCE_OPTS` as deprecated settings, if you run into issues 20 | 21 | * `/usr/local/confluent/ksql-run-class` 22 | 23 | replace 24 | 25 | ```shell 26 | KSQL_JVM_PERFORMANCE_OPTS="-server -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+ExplicitGCInvokesConcurrent -XX:NewRatio=1 -Djava.awt.headless=true" 27 | ``` 28 | 29 | with 30 | 31 | ```shell 32 | KSQL_JVM_PERFORMANCE_OPTS="-server -XX:+ExplicitGCInvokesConcurrent -XX:NewRatio=1 -Djava.awt.headless=true" 33 | ``` 34 | 35 | * Projects are being added to allow for a rich set of applications to be used in a local environment. 36 | Some are still incubating, some are well tested. 37 | 38 | # kafka 39 | 40 | * This is the `Apache Kafka Cluster` and is the core pieces needed for working with Kafka Brokers 41 | 42 | * 1 Zookeeper 43 | 44 | * 4 Brokers 45 | 46 | * 1 Schema Registry 47 | 48 | * If you are running on a machine with limited resources, considering commenting out `broker-4`. 49 | 50 | * Having 4 brokers showcases how Kafka works better, in that most topics have a replication factor of 3, 51 | and this showcases that some brokers do not have any data for a given partition of a topic. 52 | 53 | * Only one zookeeper is provided, in an actual setup we would have 3 or 5 zookeepers. As of Kafka 2.8 it is possible 54 | to explore Kafka w/out the use of Zookeeper for the quorum. 55 | 56 | * Only one schema registry is provided, in an actual setup we would have 2 for high-availability reasons. 57 | 58 | # connect 59 | 60 | * As of Kafka 2.6 (`confluent-community` 6.0) most of the bundled connectors were removed; this is great in that it allows 61 | for easier upgrading of connectors w/out having to upgrade `connect`. 62 | 63 | * Place the desired connectors (unzipped) into the ./jars directory. 64 | 65 | # monitoring 66 | 67 | * this is an optional set of containers to monitor the health of Kafka 68 | 69 | * Grafana provides a dashboard to monitoring the Kafka metrics. 70 | 71 | * `http://localhost:3000` 72 | 73 | * username: `admin` 74 | 75 | * password: `grafana` 76 | 77 | * `kowl` and `kafka-ui` are provided for kafka exploration through a UI interface. I have not used `kafka-ui`, but 78 | looks promising. I enjoy using `kowl`. Other options to use would be `akhq`, `kafdrop` and `conduktor`. 79 | 80 | 81 | # ksqlDB 82 | 83 | * a single KSQL Server 84 | 85 | * to access the command line interface have confluent community installed locally, or uncomment out the client 86 | images and use the following: 87 | 88 | ``` 89 | docker exec -it kl_ksql-client bash 90 | % ksql http://ksql-server:8088 91 | ``` 92 | 93 | # kafka-lb 94 | 95 | * ability to add a load-balancer in front of each broker. 96 | 97 | * scripts to add network latency to each broker to test various scenarios 98 | 99 | * see the "LB" advetised listener in the kafka docker-compose file for the port used for such setup 100 | 101 | # elasticsearch 102 | 103 | * provides elasticsearch and kabana containers 104 | 105 | # postgres 106 | 107 | * provides a postgres database, with the write-ahead log (wal) enabled for change-data-capture from Debezium. 108 | 109 | * see Debezium connector in connectors for accessing data with CDC. 110 | 111 | * Standard port is exposed `5432` for easy integration into developers tools. 112 | 113 | # mysql 114 | 115 | * provides a MySQL v8 database, bin-log enabled by default with v8. 116 | 117 | * see Debezium connector in connectors for accessing data with CDC. 118 | 119 | * Standard port is exposed `3306` for easy integration into developers tools. 120 | 121 | # mysql5 122 | 123 | * provides a MySQL v5 database with bin-log enabled 124 | 125 | * see Debezium connector in connectors for accessing data with CDC. 126 | 127 | * Internal `3306` port exposed as `3307` to avoid conflics with `mysql` instance. 128 | 129 | # mongo 130 | 131 | * Mongo with a single replica set created, to allow for Debezium to properly use it with change-data-capture. 132 | 133 | * The `secret.key` can be used as-is, or you can replace it with a new secret key using the `secret.sh` set. 134 | 135 | * The secret.key is required to allow for authentication and replica sets, since in a multi-node deployment, each 136 | replica would need the same secret key. 137 | 138 | # cassandra 139 | 140 | * An Apache Cassandra v4 instance. 141 | 142 | # druid 143 | 144 | * A complete setup for running Druid. 145 | Druid image is amd64, so running the provided image is rather slow. The `.env` is stood up for allowing different images, and building your own arm64 image is rather easy, details to come. 146 | 147 | * Details (and referenced project) for building arm64 based images [druid-m1](https://www.kineticedge.io/blog/druid-m1/). 148 | 149 | * All druid ports are exposed with prefixing with a `4`, such as coordinator's `8081` is `48081`. 150 | 151 | # pinot 152 | 153 | * this is a rather new container setup, but I have used to for a POC. 154 | 155 | # storage 156 | 157 | * Minio is a s3 compliant object store. Great for testing s3 connectors locally. 158 | 159 | # flink 160 | 161 | * 1 Job Manager 162 | 163 | * Please uber-jars in `./jars` to make the accessible to the job manager for scheduling. 164 | 165 | * Use `./data` folder for any output - easier to use for development and validation. 166 | 167 | * 2 Task Managers, easily scaled with `deploy.replicas`. 168 | 169 | # oracle 170 | 171 | * provides an Oracle datbase with logminer enabled 172 | 173 | * see kafka-connect-logminer in connectors for accessing data with CDC. 174 | 175 | * the Oracle container must be built, see [README](./oracle/README.md). 176 | 177 | 178 | -------------------------------------------------------------------------------- /connect/jmx_prometheus/connect-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | lowercaseOutputLabelNames: true 4 | blacklistObjectNames: 5 | # This will ignore the admin client metrics from KSQL server and will blacklist certain metrics 6 | # that do not make sense for ingestion. 7 | - "kafka.admin.client:*" 8 | #- "kafka.consumer:type=*,id=*" 9 | #- "kafka.producer:type=*,id=*" 10 | - "kafka.producer:client-id=confluent.monitoring*,*" 11 | - "kafka.*:type=kafka-metrics-count,*" 12 | rules: 13 | - pattern: "java.lang" 14 | - pattern: "kafka.connect<>(.+): (.+)" 15 | value: 1 16 | name: kafka_connect_connector_metrics 17 | labels: 18 | connector: $1 19 | $2: $3 20 | type: UNTYPED 21 | - pattern: "kafka.connect<>([^:]+)" 22 | - pattern: "kafka.connect<>([^:]+)" 23 | - pattern: "kafka.connect<>([^:]+)" 24 | 25 | kafka: 26 | connect:connector=mysql-backup,type=connector-metrics: 27 | 28 | #- pattern: "kafka.connect<>(.*): (.*)" 29 | # name: kafka_server_app_info_$2 30 | # labels: 31 | # kafka_broker: "$1" 32 | # kafka_version: "$3" 33 | # value: 1 34 | ## "kafka.schema.registry:type=app-info,id=*" 35 | #- pattern: "kafka.connect<>(.+): (.+)" 36 | # name: "kafka_connect_app_info" 37 | # value: 1 38 | # labels: 39 | # client-id: "$1" 40 | # $2: "$3" 41 | # type: UNTYPED 42 | ## kafka.connect:type=connect-worker-rebalance-metrics 43 | #- pattern: "kafka.connect<>([^:]+)" 44 | # name: "kafka_connect_connect_worker_rebalance_metrics_$1" 45 | ## kafka.connect:type=connect-coordinator-metrics,client-id=* 46 | ## kafka.connect:type=connect-metrics,client-id=* 47 | #- pattern: "kafka.connect<>([^:]+)" 48 | # name: kafka_connect_$1_$3 49 | # labels: 50 | # client_id: $2 51 | ## kafka.connect:type=connect-worker-metrics 52 | #- pattern: "kafka.connect<>([^:]+)" 53 | # name: kafka_connect_connect_worker_metrics_$1 54 | # labels: 55 | # connector: "aggregate" 56 | ## kafka.connect:type=connect-worker-metrics,connector=* 57 | #- pattern: "kafka.connect<>([^:]+)" 58 | # name: kafka_connect_connect_worker_metrics_$2 59 | # labels: 60 | # connector: $1 61 | ## kafka.connect:type=connector-metrics,connector=* 62 | #- pattern: "kafka.connect<>(.+): (.+)" 63 | # value: 1 64 | # name: kafka_connect_connector_metrics 65 | # labels: 66 | # connector: $1 67 | # $2: $3 68 | # type: UNTYPED 69 | ## kafka.connect:type=*-task-metrics,* 70 | ## kafka.connect:type=source-task-metrics,connector=*,task=* 71 | ## kafka.connect:type=sink-task-metrics,connector=*,task=* 72 | ## kafka.connect:type=connector-task-metrics,connector=*,task=* 73 | #- pattern: "kafka.connect<>(.+): (.+)" 74 | # name: kafka_connect_$1_task_metrics_$4 75 | # labels: 76 | # connector: "$2" 77 | # task: "$3" 78 | ## kafka.connect:type=task-error-metrics,* 79 | ## kafka.connect:type=task-error-metrics,connector=*,task=* 80 | #- pattern: "kafka.connect<>([^:]+)" 81 | # name: kafka_connect_task_error_metrics_$3 82 | # labels: 83 | # connector: "$1" 84 | # task: "$2" 85 | ## confluent.replicator:type=confluent-replicator-task-metrics,* : confluent-replicator-task-topic-partition-*: Number Values 86 | #- pattern: "confluent.replicator<>confluent-replicator-task-topic-partition-(.*): (.*)" 87 | # name: confluent_replicator_task_metrics_$9 88 | # labels: 89 | # $1: "$2" 90 | # $3: "$4" 91 | # $5: "$6" 92 | # $7: "$8" 93 | ## confluent.replicator:type=confluent-replicator-task-metrics,* : Strings 94 | #- pattern: "confluent.replicator<>(confluent-replicator-destination-cluster|confluent-replicator-source-cluster|confluent-replicator-destination-topic-name): (.*)" 95 | # name: confluent_replicator_task_metrics_info 96 | # value: 1 97 | # labels: 98 | # $1: "$2" 99 | # $3: "$4" 100 | # $5: "$6" 101 | # $7: "$8" 102 | # $9: "$10" 103 | ## "kafka.consumer:type=app-info,client-id=*" 104 | ## "kafka.producer:type=app-info,client-id=*" 105 | #- pattern: "kafka.(.+)<>(.+): (.+)" 106 | # value: 1 107 | # name: kafka_$1_app_info 108 | # labels: 109 | # client_type: $1 110 | # client_id: $2 111 | # $3: $4 112 | # type: UNTYPED 113 | ## "kafka.consumer:type=consumer-metrics,client-id=*, protocol=*, cipher=*" 114 | ## "kafka.consumer:type=type=consumer-fetch-manager-metrics,client-id=*, topic=*, partition=*" 115 | ## "kafka.producer:type=producer-metrics,client-id=*, protocol=*, cipher=*" 116 | #- pattern: "kafka.(.+)<>(.+):" 117 | # name: kafka_$1_$2_$9 118 | # type: GAUGE 119 | # labels: 120 | # client_type: $1 121 | # $3: "$4" 122 | # $5: "$6" 123 | # $7: "$8" 124 | ## "kafka.consumer:type=consumer-node-metrics,client-id=*, node-id=*" 125 | ## "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*, topic=*" 126 | ## "kafka.producer:type=producer-node-metrics,client-id=*, node-id=*" 127 | ## "kafka.producer:type=producer-topic-metrics,client-id=*, topic=*" 128 | #- pattern: "kafka.(.+)<>(.+):" 129 | # name: kafka_$1_$2_$7 130 | # type: GAUGE 131 | # labels: 132 | # client_type: $1 133 | # $3: "$4" 134 | # $5: "$6" 135 | ## "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*" 136 | ## "kafka.consumer:type=consumer-metrics,client-id=*" 137 | ## "kafka.producer:type=producer-metrics,client-id=*" 138 | #- pattern: "kafka.(.+)<>(.+):" 139 | # name: kafka_$1_$2_$5 140 | # type: GAUGE 141 | # labels: 142 | # client_type: $1 143 | # $3: "$4" 144 | #- pattern: "kafka.(.+)<>(.+):" 145 | # name: kafka_$1_$2_$3 146 | # labels: 147 | # client_type: $1 148 | 149 | 150 | # kafka.connect:client-id=connect-1,node-id=node--3,type=connect-node-metrics 151 | # kafka.connect:client-id=connect-1,node-id=node-2147483646,type=connect-node-metrics 152 | # kafka.connect:client-id=connect-1,type=app-info 153 | # kafka.connect:client-id=connect-1,type=connect-coordinator-metrics 154 | # kafka.connect:client-id=connect-1,type=connect-metrics 155 | # kafka.connect:client-id=connect-1,type=kafka-metrics-count 156 | # kafka.connect:connector=store,task=0,type=connector-task-metrics 157 | # kafka.connect:connector=store,task=0,type=source-task-metrics 158 | # kafka.connect:connector=store,task=0,type=task-error-metrics 159 | # kafka.connect:connector=store,type=connect-worker-metrics 160 | # kafka.connect:connector=store,type=connector-metrics 161 | # kafka.connect:id="connect-1:8083",type=app-info 162 | # kafka.connect:id=connect-1,type=app-info 163 | # kafka.connect:type=app-info 164 | # kafka.connect:type=connect-worker-metrics 165 | # kafka.connect:type=connect-worker-rebalance-metrics 166 | # kafka.connect:type=kafka-metrics-count 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /kafka-zk/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | zk: {} 8 | zk_log: {} 9 | bk1: {} 10 | bk2: {} 11 | bk3: {} 12 | bk4: {} 13 | 14 | services: 15 | 16 | zookeeper: 17 | image: confluentinc/cp-zookeeper:${VERSION} 18 | container_name: zookeeper 19 | hostname: zookeeper 20 | ports: 21 | - '2181:2181' 22 | healthcheck: 23 | test: echo ruok | nc -w 2 localhost 2181 | grep imok 24 | start_period: 5s 25 | interval: 5s 26 | timeout: 5s 27 | retries: 3 28 | environment: 29 | ZOOKEEPER_CLIENT_PORT: 2181 30 | ZOOKEEPER_TICK_TIME: 2000 31 | KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok,stat" 32 | volumes: 33 | - zk:/var/lib/zookeeper/data 34 | - zk_log:/var/lib/zookeeper/log 35 | - ./secrets:/etc/zookeeper/secrets 36 | 37 | broker-1: 38 | image: confluentinc/cp-kafka:${VERSION} 39 | hostname: broker-1 40 | container_name: broker-1 41 | stop_grace_period: 120s 42 | healthcheck: 43 | test: nc -z localhost 9092 || exit 1 44 | start_period: 5s 45 | interval: 5s 46 | timeout: 5s 47 | retries: 3 48 | depends_on: 49 | zookeeper: 50 | condition: service_healthy 51 | ports: 52 | # 9092 can only be used for broker discovery, 19092 will be actual returned port for this broker 53 | - '9092:19092' 54 | - '19092:19092' 55 | - '17071:7071' 56 | volumes: 57 | - bk1:/var/lib/kafka/data 58 | - ./jmx_prometheus:/opt/jmx_prometheus 59 | - ./secrets:/etc/kafka/secrets 60 | environment: 61 | KAFKA_BROKER_ID: 1 62 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 63 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 64 | # LB is an optional listener that is fronted by an nginx loadbalancer allowing for testing cluster with artificial network latency. 65 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,LB:PLAINTEXT 66 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:9092,PLAINTEXT_HOST://localhost:19092,LB://broker-1-lb:9094 67 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: true 68 | #KAFKA_DELETE_TOPIC_ENABLE: false 69 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 70 | KAFKA_MIN_INSYNC_REPLICAS: 2 71 | KAFKA_NUM_PARTITIONS: 4 72 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 73 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 74 | KAFKA_METRIC_REPORTERS: 75 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 76 | CONFLUENT_METRICS_ENABLE: 'false' 77 | 78 | 79 | broker-2: 80 | image: confluentinc/cp-kafka:${VERSION} 81 | hostname: broker-2 82 | container_name: broker-2 83 | stop_grace_period: 120s 84 | healthcheck: 85 | test: nc -z localhost 9092 || exit 1 86 | start_period: 5s 87 | interval: 5s 88 | timeout: 5s 89 | retries: 3 90 | depends_on: 91 | zookeeper: 92 | condition: service_healthy 93 | ports: 94 | - '29092:29092' 95 | volumes: 96 | - bk2:/var/lib/kafka/data 97 | - ./jmx_prometheus:/opt/jmx_prometheus 98 | - ./secrets:/etc/kafka/secrets 99 | environment: 100 | KAFKA_BROKER_ID: 2 101 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 102 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 103 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,LB:PLAINTEXT 104 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:9092,PLAINTEXT_HOST://localhost:29092,LB://broker-2-lb:9094 105 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: true 106 | #KAFKA_DELETE_TOPIC_ENABLE: false 107 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 108 | KAFKA_MIN_INSYNC_REPLICAS: 2 109 | KAFKA_NUM_PARTITIONS: 4 110 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 111 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 112 | KAFKA_METRIC_REPORTERS: 113 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 114 | CONFLUENT_METRICS_ENABLE: 'false' 115 | 116 | broker-3: 117 | image: confluentinc/cp-kafka:${VERSION} 118 | hostname: broker-3 119 | container_name: broker-3 120 | stop_grace_period: 120s 121 | healthcheck: 122 | test: nc -z localhost 9092 || exit 1 123 | start_period: 5s 124 | interval: 5s 125 | timeout: 5s 126 | retries: 3 127 | depends_on: 128 | zookeeper: 129 | condition: service_healthy 130 | ports: 131 | - '39092:39092' 132 | volumes: 133 | - bk3:/var/lib/kafka/data 134 | - ./jmx_prometheus:/opt/jmx_prometheus 135 | - ./secrets:/etc/kafka/secrets 136 | environment: 137 | KAFKA_BROKER_ID: 3 138 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 139 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 140 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,LB:PLAINTEXT 141 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:9092,PLAINTEXT_HOST://localhost:39092,LB://broker-3-lb:9094 142 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: true 143 | #KAFKA_DELETE_TOPIC_ENABLE: false 144 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 145 | KAFKA_MIN_INSYNC_REPLICAS: 2 146 | KAFKA_NUM_PARTITIONS: 4 147 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 148 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 149 | KAFKA_METRIC_REPORTERS: 150 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 151 | CONFLUENT_METRICS_ENABLE: 'false' 152 | 153 | broker-4: 154 | image: confluentinc/cp-kafka:${VERSION} 155 | hostname: broker-4 156 | container_name: broker-4 157 | stop_grace_period: 120s 158 | healthcheck: 159 | test: nc -z localhost 9092 || exit 1 160 | start_period: 5s 161 | interval: 5s 162 | timeout: 5s 163 | retries: 3 164 | depends_on: 165 | zookeeper: 166 | condition: service_healthy 167 | ports: 168 | - '49092:49092' 169 | volumes: 170 | - bk4:/var/lib/kafka/data 171 | - ./jmx_prometheus:/opt/jmx_prometheus 172 | - ./secrets:/etc/kafka/secrets 173 | environment: 174 | KAFKA_BROKER_ID: 4 175 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 176 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 177 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,LB:PLAINTEXT 178 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-4:9092,PLAINTEXT_HOST://localhost:49092,LB://broker-4-lb:9094 179 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: true 180 | #KAFKA_DELETE_TOPIC_ENABLE: false 181 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 182 | KAFKA_MIN_INSYNC_REPLICAS: 2 183 | KAFKA_NUM_PARTITIONS: 4 184 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 185 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 186 | KAFKA_METRIC_REPORTERS: 187 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 188 | CONFLUENT_METRICS_ENABLE: 'false' 189 | 190 | schema-registry: 191 | image: confluentinc/cp-schema-registry:${VERSION} 192 | hostname: schema-registry 193 | container_name: schema-registry 194 | stop_grace_period: 120s 195 | healthcheck: 196 | test: curl --fail http://localhost:8081/ || exit 1 197 | start_period: 5s 198 | interval: 5s 199 | timeout: 5s 200 | retries: 3 201 | depends_on: 202 | broker-1: 203 | condition: service_healthy 204 | broker-2: 205 | condition: service_healthy 206 | broker-3: 207 | condition: service_healthy 208 | ports: 209 | - '8081:8081' 210 | volumes: 211 | - ./jmx_prometheus:/opt/jmx_prometheus 212 | - ./secrets:/etc/schema-registry/secrets 213 | environment: 214 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 215 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092 216 | KAFKA_METRIC_REPORTERS: 217 | CONFLUENT_METRICS_ENABLE: 'false' 218 | EXTRA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/sr-config.yml 219 | SCHEMA_REGISTRY_OPTS : -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/sr-config.yml 220 | -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/streams/streams_process_blank.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "iteration": 1614565749284, 19 | "links": [], 20 | "panels": [], 21 | "refresh": "1s", 22 | "schemaVersion": 27, 23 | "style": "dark", 24 | "tags": [ 25 | ], 26 | "templating": { 27 | "list": [ 28 | { 29 | "allValue": null, 30 | "current": { 31 | "selected": false, 32 | "text": "streams", 33 | "value": "streams" 34 | }, 35 | "datasource": "prometheus", 36 | "definition": "kafka_streams_stream_thread_metrics_task_created_total", 37 | "description": null, 38 | "error": null, 39 | "hide": 0, 40 | "includeAll": true, 41 | "label": null, 42 | "multi": false, 43 | "name": "job", 44 | "options": [], 45 | "query": { 46 | "query": "kafka_streams_stream_thread_metrics_task_created_total", 47 | "refId": "prometheus-job-Variable-Query" 48 | }, 49 | "refresh": 1, 50 | "regex": "/job=\\\"(.*)\\\",/", 51 | "skipUrlSync": false, 52 | "sort": 0, 53 | "tagValuesQuery": "", 54 | "tags": [], 55 | "tagsQuery": "", 56 | "type": "query", 57 | "useTags": false 58 | }, 59 | { 60 | "allValue": null, 61 | "current": { 62 | "selected": false, 63 | "text": "All", 64 | "value": "$__all" 65 | }, 66 | "datasource": "prometheus", 67 | "definition": "kafka_streams_stream_thread_metrics_process_rate", 68 | "description": null, 69 | "error": null, 70 | "hide": 0, 71 | "includeAll": true, 72 | "label": "instance", 73 | "multi": false, 74 | "name": "instance", 75 | "options": [], 76 | "query": { 77 | "query": "kafka_streams_stream_thread_metrics_poll_rate{job=~\"$job\"}", 78 | "refId": "StandardVariableQuery" 79 | }, 80 | "refresh": 1, 81 | "regex": "/.*instance=\"(.*)\",job.*$/", 82 | "skipUrlSync": false, 83 | "sort": 0, 84 | "tagValuesQuery": "", 85 | "tags": [], 86 | "tagsQuery": "", 87 | "type": "query", 88 | "useTags": false 89 | }, 90 | { 91 | "allValue": null, 92 | "current": { 93 | "selected": false, 94 | "text": "All", 95 | "value": "$__all" 96 | }, 97 | "datasource": "prometheus", 98 | "definition": "kafka_streams_stream_thread_metrics_task_created_total{instance=~\"${instance}\"}", 99 | "description": null, 100 | "error": null, 101 | "hide": 0, 102 | "includeAll": true, 103 | "label": "thread", 104 | "multi": false, 105 | "name": "thread", 106 | "options": [], 107 | "query": { 108 | "query": "kafka_streams_stream_thread_metrics_task_created_total{instance=~\"${instance}\"}", 109 | "refId": "StandardVariableQuery" 110 | }, 111 | "refresh": 1, 112 | "regex": "/.*thread_id=\"(.*)\"}$/", 113 | "skipUrlSync": false, 114 | "sort": 0, 115 | "tagValuesQuery": "", 116 | "tags": [], 117 | "tagsQuery": "", 118 | "type": "query", 119 | "useTags": false 120 | }, 121 | { 122 | "allValue": null, 123 | "current": { 124 | "selected": true, 125 | "text": "All", 126 | "value": "$__all" 127 | }, 128 | "datasource": "prometheus", 129 | "definition": "kafka_streams_stream_processor_node_metrics_process_total{instance=~\"${instance}\",thread_id=~\"${thread}\"}", 130 | "description": null, 131 | "error": null, 132 | "hide": 0, 133 | "includeAll": true, 134 | "label": "task", 135 | "multi": false, 136 | "name": "task", 137 | "options": [ 138 | { 139 | "selected": true, 140 | "text": "All", 141 | "value": "$__all" 142 | }, 143 | { 144 | "selected": false, 145 | "text": "1", 146 | "value": "1" 147 | }, 148 | { 149 | "selected": false, 150 | "text": "2", 151 | "value": "2" 152 | }, 153 | { 154 | "selected": false, 155 | "text": "3", 156 | "value": "3" 157 | }, 158 | { 159 | "selected": false, 160 | "text": "4", 161 | "value": "4" 162 | }, 163 | { 164 | "selected": false, 165 | "text": "5", 166 | "value": "5" 167 | } 168 | ], 169 | "query": { 170 | "query": "kafka_streams_stream_processor_node_metrics_process_total{instance=~\"${instance}\",thread_id=~\"${thread}\"}", 171 | "refId": "StandardVariableQuery" 172 | }, 173 | "refresh": 0, 174 | "regex": "/.*task_id=\"([0-9]+)\",.*$/", 175 | "skipUrlSync": false, 176 | "sort": 3, 177 | "tagValuesQuery": "", 178 | "tags": [], 179 | "tagsQuery": "", 180 | "type": "query", 181 | "useTags": false 182 | }, 183 | { 184 | "allValue": ".*", 185 | "current": { 186 | "selected": true, 187 | "text": "pickup-orders-consumed", 188 | "value": "pickup-orders-consumed" 189 | }, 190 | "datasource": "prometheus", 191 | "definition": "kafka_streams_stream_processor_node_metrics_process_total{task_id=~\"$task\",thread_id=~\"${thread}\",instance=~\"$instance\",job=~\"$job\"}\n", 192 | "description": null, 193 | "error": null, 194 | "hide": 0, 195 | "includeAll": true, 196 | "label": "process", 197 | "multi": false, 198 | "name": "process", 199 | "options": [], 200 | "query": { 201 | "query": "kafka_streams_stream_processor_node_metrics_process_total{task_id=~\"$task\",thread_id=~\"${thread}\",instance=~\"$instance\",job=~\"$job\"}\n", 202 | "refId": "StandardVariableQuery" 203 | }, 204 | "refresh": 1, 205 | "regex": "/.*processor_node_id=\"(.*)\",task_id=.*$/", 206 | "skipUrlSync": false, 207 | "sort": 3, 208 | "tagValuesQuery": "", 209 | "tags": [], 210 | "tagsQuery": "", 211 | "type": "query", 212 | "useTags": true 213 | }, 214 | { 215 | "allValue": "", 216 | "current": { 217 | "selected": false, 218 | "text": "All", 219 | "value": "$__all" 220 | }, 221 | "datasource": "prometheus", 222 | "definition": "kafka_streams_stream_task_metrics_process_rate{instance=~\"${instance}\",thread_id=~\"${thread}\"}", 223 | "description": null, 224 | "error": null, 225 | "hide": 0, 226 | "includeAll": true, 227 | "label": "partition", 228 | "multi": false, 229 | "name": "partition", 230 | "options": [], 231 | "query": { 232 | "query": "kafka_streams_stream_task_metrics_process_rate{instance=~\"${instance}\",thread_id=~\"${thread}\"}", 233 | "refId": "StandardVariableQuery" 234 | }, 235 | "refresh": 1, 236 | "regex": "/.*partition_id=\"([0-9]+)\",tas.*$/", 237 | "skipUrlSync": false, 238 | "sort": 3, 239 | "tagValuesQuery": "", 240 | "tags": [], 241 | "tagsQuery": "", 242 | "type": "query", 243 | "useTags": true 244 | } 245 | ] 246 | }, 247 | "time": { 248 | "from": "now-1m", 249 | "to": "now" 250 | }, 251 | "timepicker": { 252 | "refresh_intervals": [ 253 | "2s", 254 | "5s", 255 | "10s", 256 | "30s", 257 | "1m", 258 | "5m", 259 | "15m", 260 | "30m", 261 | "1h", 262 | "2h", 263 | "1d" 264 | ], 265 | "time_options": [ 266 | "5m", 267 | "15m", 268 | "1h", 269 | "6h", 270 | "12h", 271 | "24h", 272 | "2d", 273 | "7d", 274 | "30d" 275 | ] 276 | }, 277 | "timezone": "", 278 | "title": "Stream Processes (Blank)", 279 | "uid": "streams_process_blank", 280 | "version": 1 281 | } -------------------------------------------------------------------------------- /kafka/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | ct0: {} 8 | bk1: {} 9 | bk2: {} 10 | bk3: {} 11 | bk4: {} 12 | 13 | services: 14 | controller-0: 15 | image: confluentinc/cp-kafka:${VERSION} 16 | hostname: controller-0 17 | container_name: controller-0 18 | stop_grace_period: 60s 19 | healthcheck: 20 | test: nc -z localhost 9093 || exit 1 21 | start_period: 5s 22 | interval: 5s 23 | timeout: 5s 24 | retries: 3 25 | volumes: 26 | - ct0:/var/lib/kafka/data 27 | - ./jmx_prometheus:/opt/jmx_prometheus 28 | - ./secrets:/etc/kafka/secrets 29 | environment: 30 | CLUSTER_ID: "0000000000000000000000" 31 | KAFKA_LISTENERS: CONTROLLER://:9093 32 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT 33 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 34 | KAFKA_PROCESS_ROLES: controller 35 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093 36 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 37 | KAFKA_NUM_PARTITIONS: 4 38 | KAFKA_NODE_ID: 10 39 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 40 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 41 | 42 | broker-1: 43 | image: confluentinc/cp-kafka:${VERSION} 44 | hostname: broker-1 45 | container_name: broker-1 46 | stop_grace_period: 60s 47 | healthcheck: 48 | test: nc -z localhost 9092 || exit 1 49 | start_period: 5s 50 | interval: 5s 51 | timeout: 5s 52 | retries: 3 53 | depends_on: 54 | controller-0: 55 | condition: service_healthy 56 | ports: 57 | - "19092:19092" 58 | - "9092:19092" 59 | - "17071:7071" 60 | volumes: 61 | - bk1:/var/lib/kafka/data 62 | - ./jmx_prometheus:/opt/jmx_prometheus 63 | - ./secrets:/etc/kafka/secrets 64 | environment: 65 | CLUSTER_ID: "0000000000000000000000" 66 | KAFKA_NODE_ID: 1 67 | KAFKA_PROCESS_ROLES: broker 68 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 69 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 70 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:9092,PLAINTEXT_HOST://localhost:19092 71 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093 72 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 73 | KAFKA_MIN_INSYNC_REPLICAS: 2 74 | KAFKA_NUM_PARTITIONS: 4 75 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 76 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 77 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 78 | KAFKA_METRIC_REPORTERS: 79 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 80 | CONFLUENT_METRICS_ENABLE: "false" 81 | 82 | broker-2: 83 | image: confluentinc/cp-kafka:${VERSION} 84 | hostname: broker-2 85 | container_name: broker-2 86 | stop_grace_period: 60s 87 | healthcheck: 88 | test: nc -z localhost 9092 || exit 1 89 | start_period: 5s 90 | interval: 5s 91 | timeout: 5s 92 | retries: 3 93 | depends_on: 94 | controller-0: 95 | condition: service_healthy 96 | ports: 97 | - "29092:29092" 98 | - "27071:7071" 99 | volumes: 100 | - bk2:/var/lib/kafka/data 101 | - ./jmx_prometheus:/opt/jmx_prometheus 102 | - ./secrets:/etc/kafka/secrets 103 | environment: 104 | CLUSTER_ID: "0000000000000000000000" 105 | KAFKA_NODE_ID: 2 106 | KAFKA_PROCESS_ROLES: broker 107 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 108 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 109 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:9092,PLAINTEXT_HOST://localhost:29092 110 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093 111 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 112 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 113 | KAFKA_MIN_INSYNC_REPLICAS: 2 114 | KAFKA_NUM_PARTITIONS: 4 115 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 116 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 117 | KAFKA_METRIC_REPORTERS: 118 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 119 | CONFLUENT_METRICS_ENABLE: "false" 120 | 121 | broker-3: 122 | image: confluentinc/cp-kafka:${VERSION} 123 | hostname: broker-3 124 | container_name: broker-3 125 | stop_grace_period: 60s 126 | healthcheck: 127 | test: nc -z localhost 9092 || exit 1 128 | start_period: 5s 129 | interval: 5s 130 | timeout: 5s 131 | retries: 3 132 | depends_on: 133 | controller-0: 134 | condition: service_healthy 135 | ports: 136 | - "39092:39092" 137 | - "37071:7071" 138 | volumes: 139 | - bk3:/var/lib/kafka/data 140 | - ./jmx_prometheus:/opt/jmx_prometheus 141 | - ./secrets:/etc/kafka/secrets 142 | environment: 143 | CLUSTER_ID: "0000000000000000000000" 144 | KAFKA_NODE_ID: 3 145 | KAFKA_PROCESS_ROLES: broker 146 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 147 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 148 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:9092,PLAINTEXT_HOST://localhost:39092 149 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093 150 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 151 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 152 | KAFKA_MIN_INSYNC_REPLICAS: 2 153 | KAFKA_NUM_PARTITIONS: 4 154 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 155 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 156 | KAFKA_METRIC_REPORTERS: 157 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 158 | CONFLUENT_METRICS_ENABLE: "false" 159 | 160 | broker-4: 161 | image: confluentinc/cp-kafka:${VERSION} 162 | hostname: broker-4 163 | container_name: broker-4 164 | stop_grace_period: 60s 165 | healthcheck: 166 | test: nc -z localhost 9092 || exit 1 167 | start_period: 5s 168 | interval: 5s 169 | timeout: 5s 170 | retries: 3 171 | depends_on: 172 | controller-0: 173 | condition: service_healthy 174 | ports: 175 | - "49092:49092" 176 | - "47071:7071" 177 | volumes: 178 | - bk4:/var/lib/kafka/data 179 | - ./jmx_prometheus:/opt/jmx_prometheus 180 | - ./secrets:/etc/kafka/secrets 181 | environment: 182 | CLUSTER_ID: "0000000000000000000000" 183 | KAFKA_NODE_ID: 4 184 | KAFKA_PROCESS_ROLES: broker 185 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 186 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 187 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-4:9092,PLAINTEXT_HOST://localhost:49092 188 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093 189 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 190 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 191 | KAFKA_MIN_INSYNC_REPLICAS: 2 192 | KAFKA_NUM_PARTITIONS: 4 193 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 194 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 195 | KAFKA_METRIC_REPORTERS: 196 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 197 | CONFLUENT_METRICS_ENABLE: "false" 198 | 199 | kraft-schema-registry: 200 | image: confluentinc/cp-schema-registry:${VERSION} 201 | hostname: schema-registry 202 | container_name: schema-registry 203 | stop_grace_period: 60s 204 | healthcheck: 205 | test: curl --fail http://localhost:8081/ || exit 1 206 | start_period: 5s 207 | interval: 5s 208 | timeout: 5s 209 | retries: 3 210 | depends_on: 211 | controller-0: 212 | condition: service_healthy 213 | broker-1: 214 | condition: service_healthy 215 | broker-2: 216 | condition: service_healthy 217 | broker-3: 218 | condition: service_healthy 219 | ports: 220 | - "8081:8081" 221 | volumes: 222 | - ./jmx_prometheus:/opt/jmx_prometheus 223 | - ./secrets:/etc/schema-registry/secrets 224 | environment: 225 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 226 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092 227 | KAFKA_METRIC_REPORTERS: 228 | CONFLUENT_METRICS_ENABLE: "false" 229 | EXTRA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/sr-config.yml 230 | SCHEMA_REGISTRY_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/sr-config.yml 231 | 232 | # rest-proxy: 233 | # image: confluentinc/cp-kafka-rest:${VERSION} 234 | # hostname: rest-proxy 235 | # container_name: rest-proxy 236 | # ports: 237 | # - 8082:8082 238 | # environment: 239 | # KAFKA_REST_BOOTSTRAP_SERVERS: "PLAINTEXT://broker-1:9092" 240 | # KAFKA_REST_HOST_NAME: rest-proxy 241 | # KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" 242 | # KAFKA_REST_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" 243 | -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/kafka/11_kafka_cluster_request_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "datasource", 8 | "uid": "grafana" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "links": [], 28 | "liveNow": false, 29 | "panels": [ 30 | { 31 | "datasource": { 32 | "type": "prometheus", 33 | "uid": "prometheus" 34 | }, 35 | "fieldConfig": { 36 | "defaults": { 37 | "color": { 38 | "mode": "palette-classic" 39 | }, 40 | "custom": { 41 | "axisCenteredZero": false, 42 | "axisColorMode": "text", 43 | "axisLabel": "requests/s", 44 | "axisPlacement": "auto", 45 | "barAlignment": 0, 46 | "drawStyle": "line", 47 | "fillOpacity": 10, 48 | "gradientMode": "none", 49 | "hideFrom": { 50 | "legend": false, 51 | "tooltip": false, 52 | "viz": false 53 | }, 54 | "lineInterpolation": "linear", 55 | "lineWidth": 2, 56 | "pointSize": 5, 57 | "scaleDistribution": { 58 | "type": "linear" 59 | }, 60 | "showPoints": "never", 61 | "spanNulls": false, 62 | "stacking": { 63 | "group": "A", 64 | "mode": "none" 65 | }, 66 | "thresholdsStyle": { 67 | "mode": "off" 68 | } 69 | }, 70 | "decimals": 2, 71 | "mappings": [], 72 | "thresholds": { 73 | "mode": "absolute", 74 | "steps": [ 75 | { 76 | "color": "green", 77 | "value": null 78 | }, 79 | { 80 | "color": "red", 81 | "value": 80 82 | } 83 | ] 84 | }, 85 | "unit": "none" 86 | }, 87 | "overrides": [] 88 | }, 89 | "gridPos": { 90 | "h": 12, 91 | "w": 12, 92 | "x": 0, 93 | "y": 0 94 | }, 95 | "id": 2, 96 | "interval": "5s", 97 | "links": [], 98 | "options": { 99 | "legend": { 100 | "calcs": [ 101 | "lastNotNull" 102 | ], 103 | "displayMode": "table", 104 | "placement": "bottom", 105 | "showLegend": true, 106 | "sortBy": "Last *", 107 | "sortDesc": true 108 | }, 109 | "tooltip": { 110 | "mode": "single", 111 | "sort": "none" 112 | } 113 | }, 114 | "pluginVersion": "8.4.0-46516pre", 115 | "targets": [ 116 | { 117 | "datasource": { 118 | "type": "prometheus", 119 | "uid": "prometheus" 120 | }, 121 | "editorMode": "code", 122 | "expr": "sum without(instance, version) (kafka_network_requestmetrics_oneminuterate{cluster_id=~\"$cluster\", name=\"RequestsPerSec\", instance=~\"$instance\"}) ", 123 | "format": "time_series", 124 | "intervalFactor": 2, 125 | "legendFormat": "{{request}}", 126 | "range": true, 127 | "refId": "C" 128 | } 129 | ], 130 | "title": "Request Metrics", 131 | "type": "timeseries" 132 | }, 133 | { 134 | "datasource": { 135 | "type": "prometheus", 136 | "uid": "prometheus" 137 | }, 138 | "fieldConfig": { 139 | "defaults": { 140 | "color": { 141 | "mode": "palette-classic" 142 | }, 143 | "custom": { 144 | "axisCenteredZero": false, 145 | "axisColorMode": "text", 146 | "axisGridShow": true, 147 | "axisLabel": "count", 148 | "axisPlacement": "auto", 149 | "barAlignment": 0, 150 | "drawStyle": "line", 151 | "fillOpacity": 10, 152 | "gradientMode": "none", 153 | "hideFrom": { 154 | "legend": false, 155 | "tooltip": false, 156 | "viz": false 157 | }, 158 | "lineInterpolation": "linear", 159 | "lineWidth": 2, 160 | "pointSize": 5, 161 | "scaleDistribution": { 162 | "type": "linear" 163 | }, 164 | "showPoints": "never", 165 | "spanNulls": false, 166 | "stacking": { 167 | "group": "A", 168 | "mode": "none" 169 | }, 170 | "thresholdsStyle": { 171 | "mode": "off" 172 | } 173 | }, 174 | "mappings": [], 175 | "thresholds": { 176 | "mode": "absolute", 177 | "steps": [ 178 | { 179 | "color": "green", 180 | "value": null 181 | }, 182 | { 183 | "color": "red", 184 | "value": 80 185 | } 186 | ] 187 | }, 188 | "unit": "none" 189 | }, 190 | "overrides": [ 191 | { 192 | "__systemRef": "hideSeriesFrom", 193 | "matcher": { 194 | "id": "byNames", 195 | "options": { 196 | "mode": "exclude", 197 | "names": [ 198 | "Produce" 199 | ], 200 | "prefix": "All except:", 201 | "readOnly": true 202 | } 203 | }, 204 | "properties": [ 205 | { 206 | "id": "custom.hideFrom", 207 | "value": { 208 | "legend": false, 209 | "tooltip": false, 210 | "viz": true 211 | } 212 | } 213 | ] 214 | } 215 | ] 216 | }, 217 | "gridPos": { 218 | "h": 12, 219 | "w": 12, 220 | "x": 12, 221 | "y": 0 222 | }, 223 | "id": 13, 224 | "interval": "5s", 225 | "links": [], 226 | "options": { 227 | "legend": { 228 | "calcs": [ 229 | "lastNotNull" 230 | ], 231 | "displayMode": "table", 232 | "placement": "bottom", 233 | "showLegend": true 234 | }, 235 | "tooltip": { 236 | "mode": "single", 237 | "sort": "none" 238 | } 239 | }, 240 | "targets": [ 241 | { 242 | "datasource": { 243 | "type": "prometheus", 244 | "uid": "prometheus" 245 | }, 246 | "editorMode": "code", 247 | "expr": " (kafka_network_requestchannel_value{cluster_id=~\"$cluster\", instance=~\"$instance\", processor=\"\"}) ", 248 | "format": "time_series", 249 | "intervalFactor": 2, 250 | "legendFormat": "__auto", 251 | "range": true, 252 | "refId": "C" 253 | } 254 | ], 255 | "title": "RequestsPerSec", 256 | "type": "timeseries" 257 | } 258 | ], 259 | "refresh": "2s", 260 | "schemaVersion": 37, 261 | "style": "dark", 262 | "tags": [ 263 | "kafka_cluster", 264 | "kafka" 265 | ], 266 | "templating": { 267 | "list": [ 268 | { 269 | "allValue": ".+", 270 | "current": { 271 | "selected": false, 272 | "text": "kafka-core", 273 | "value": "kafka-core" 274 | }, 275 | "datasource": { 276 | "type": "prometheus", 277 | "uid": "prometheus" 278 | }, 279 | "definition": "label_values(jvm_info{cluster_type=\"kafka\"}, cluster_id)", 280 | "hide": 0, 281 | "includeAll": false, 282 | "label": "cluster", 283 | "multi": false, 284 | "name": "cluster", 285 | "options": [], 286 | "query": { 287 | "query": "label_values(jvm_info{cluster_type=\"kafka\"}, cluster_id)", 288 | "refId": "StandardVariableQuery" 289 | }, 290 | "refresh": 1, 291 | "regex": "", 292 | "skipUrlSync": false, 293 | "sort": 0, 294 | "tagValuesQuery": "", 295 | "tagsQuery": "", 296 | "type": "query", 297 | "useTags": false 298 | }, 299 | { 300 | "allValue": ".*", 301 | "current": { 302 | "selected": true, 303 | "text": "All", 304 | "value": "$__all" 305 | }, 306 | "datasource": { 307 | "type": "prometheus", 308 | "uid": "prometheus" 309 | }, 310 | "definition": "label_values(jvm_info{cluster_type=\"kafka\", cluster_id=\"$cluster\"}, instance)", 311 | "hide": 0, 312 | "includeAll": true, 313 | "label": "instance", 314 | "multi": false, 315 | "name": "instance", 316 | "options": [], 317 | "query": { 318 | "query": "label_values(jvm_info{cluster_type=\"kafka\", cluster_id=\"$cluster\"}, instance)", 319 | "refId": "StandardVariableQuery" 320 | }, 321 | "refresh": 1, 322 | "regex": "", 323 | "skipUrlSync": false, 324 | "sort": 0, 325 | "tagValuesQuery": "", 326 | "tagsQuery": "", 327 | "type": "query", 328 | "useTags": false 329 | } 330 | ] 331 | }, 332 | "time": { 333 | "from": "now-1m", 334 | "to": "now" 335 | }, 336 | "timepicker": { 337 | "refresh_intervals": [ 338 | "2s", 339 | "5s", 340 | "10s", 341 | "30s", 342 | "1m", 343 | "5m", 344 | "15m", 345 | "30m", 346 | "1h", 347 | "2h", 348 | "1d" 349 | ], 350 | "time_options": [ 351 | "1m", 352 | "5m", 353 | "15m", 354 | "1h", 355 | "6h", 356 | "12h", 357 | "24h", 358 | "2d", 359 | "7d", 360 | "30d" 361 | ] 362 | }, 363 | "timezone": "", 364 | "title": "11 - Request Metrics 2", 365 | "uid": "kafka_cluster_request_2", 366 | "version": 1, 367 | "weekStart": "" 368 | } -------------------------------------------------------------------------------- /kafka-raft/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: dev-local 5 | 6 | volumes: 7 | kraft-ct0: {} 8 | kraft-ct1: {} 9 | kraft-ct2: {} 10 | kraft-bk1: {} 11 | kraft-bk2: {} 12 | kraft-bk3: {} 13 | kraft-bk4: {} 14 | 15 | services: 16 | 17 | controller-0: 18 | image: confluentinc/cp-kafka:${VERSION} 19 | hostname: controller-0 20 | container_name: kraft-controller-0 21 | stop_grace_period: 60s 22 | healthcheck: 23 | test: nc -z localhost 9093 || exit 1 24 | start_period: 5s 25 | interval: 5s 26 | timeout: 5s 27 | retries: 3 28 | volumes: 29 | - kraft-ct0:/var/lib/kafka/data 30 | - ./jmx_prometheus:/opt/jmx_prometheus 31 | - ./secrets:/etc/kafka/secrets 32 | environment: 33 | CLUSTER_ID: "0000000000000000000000" 34 | KAFKA_LISTENERS: CONTROLLER://:9093 35 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT 36 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 37 | KAFKA_PROCESS_ROLES: controller 38 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 39 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 40 | KAFKA_NUM_PARTITIONS: 4 41 | KAFKA_NODE_ID: 10 42 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 43 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 44 | 45 | controller-1: 46 | image: confluentinc/cp-kafka:${VERSION} 47 | hostname: controller-1 48 | container_name: kraft-controller-1 49 | stop_grace_period: 60s 50 | healthcheck: 51 | test: nc -z localhost 9093 || exit 1 52 | start_period: 5s 53 | interval: 5s 54 | timeout: 5s 55 | retries: 3 56 | volumes: 57 | - kraft-ct1:/var/lib/kafka/data 58 | - ./jmx_prometheus:/opt/jmx_prometheus 59 | - ./secrets:/etc/kafka/secrets 60 | environment: 61 | CLUSTER_ID: "0000000000000000000000" 62 | KAFKA_LISTENERS: CONTROLLER://:9093 63 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT 64 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 65 | KAFKA_PROCESS_ROLES: controller 66 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 67 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 68 | KAFKA_NUM_PARTITIONS: 4 69 | KAFKA_NODE_ID: 11 70 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 71 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 72 | 73 | controller-2: 74 | image: confluentinc/cp-kafka:${VERSION} 75 | hostname: controller-2 76 | container_name: kraft-controller-2 77 | stop_grace_period: 60s 78 | healthcheck: 79 | test: nc -z localhost 9093 || exit 1 80 | start_period: 5s 81 | interval: 5s 82 | timeout: 5s 83 | retries: 3 84 | volumes: 85 | - kraft-ct2:/var/lib/kafka/data 86 | - ./jmx_prometheus:/opt/jmx_prometheus 87 | - ./secrets:/etc/kafka/secrets 88 | environment: 89 | CLUSTER_ID: "0000000000000000000000" 90 | KAFKA_LISTENERS: CONTROLLER://:9093 91 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT 92 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 93 | KAFKA_PROCESS_ROLES: controller 94 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 95 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 96 | KAFKA_NUM_PARTITIONS: 4 97 | KAFKA_NODE_ID: 12 98 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 99 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 100 | 101 | 102 | broker-1: 103 | image: confluentinc/cp-kafka:${VERSION} 104 | hostname: broker-1 105 | container_name: kraft-broker-1 106 | stop_grace_period: 60s 107 | healthcheck: 108 | test: nc -z localhost 9092 || exit 1 109 | start_period: 5s 110 | interval: 5s 111 | timeout: 5s 112 | retries: 3 113 | depends_on: 114 | controller-0: 115 | condition: service_healthy 116 | controller-1: 117 | condition: service_healthy 118 | controller-2: 119 | condition: service_healthy 120 | ports: 121 | - '19092:19092' 122 | - '17071:7071' 123 | volumes: 124 | - kraft-bk1:/var/lib/kafka/data 125 | - ./jmx_prometheus:/opt/jmx_prometheus 126 | - ./secrets:/etc/kafka/secrets 127 | environment: 128 | CLUSTER_ID: "0000000000000000000000" 129 | KAFKA_NODE_ID: 1 130 | KAFKA_PROCESS_ROLES: broker 131 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 132 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 133 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:9092,PLAINTEXT_HOST://localhost:19092 134 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 135 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 136 | KAFKA_MIN_INSYNC_REPLICAS: 2 137 | KAFKA_NUM_PARTITIONS: 4 138 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 139 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 140 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 141 | KAFKA_METRIC_REPORTERS: 142 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 143 | CONFLUENT_METRICS_ENABLE: 'false' 144 | 145 | broker-2: 146 | image: confluentinc/cp-kafka:${VERSION} 147 | hostname: broker-2 148 | container_name: kraft-broker-2 149 | stop_grace_period: 60s 150 | healthcheck: 151 | test: nc -z localhost 9092 || exit 1 152 | start_period: 5s 153 | interval: 5s 154 | timeout: 5s 155 | retries: 3 156 | depends_on: 157 | controller-0: 158 | condition: service_healthy 159 | controller-1: 160 | condition: service_healthy 161 | controller-2: 162 | condition: service_healthy 163 | ports: 164 | - '29092:29092' 165 | - '27071:7071' 166 | volumes: 167 | - kraft-bk2:/var/lib/kafka/data 168 | - ./jmx_prometheus:/opt/jmx_prometheus 169 | - ./secrets:/etc/kafka/secrets 170 | environment: 171 | CLUSTER_ID: "0000000000000000000000" 172 | KAFKA_NODE_ID: 2 173 | KAFKA_PROCESS_ROLES: broker 174 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 175 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 176 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:9092,PLAINTEXT_HOST://localhost:29092 177 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 178 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 179 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 180 | KAFKA_MIN_INSYNC_REPLICAS: 2 181 | KAFKA_NUM_PARTITIONS: 4 182 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 183 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 184 | KAFKA_METRIC_REPORTERS: 185 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 186 | CONFLUENT_METRICS_ENABLE: 'false' 187 | 188 | broker-3: 189 | image: confluentinc/cp-kafka:${VERSION} 190 | hostname: broker-3 191 | container_name: kraft-broker-3 192 | stop_grace_period: 60s 193 | healthcheck: 194 | test: nc -z localhost 9092 || exit 1 195 | start_period: 5s 196 | interval: 5s 197 | timeout: 5s 198 | retries: 3 199 | depends_on: 200 | controller-0: 201 | condition: service_healthy 202 | controller-1: 203 | condition: service_healthy 204 | controller-2: 205 | condition: service_healthy 206 | ports: 207 | - '39092:39092' 208 | - '37071:7071' 209 | volumes: 210 | - kraft-bk3:/var/lib/kafka/data 211 | - ./jmx_prometheus:/opt/jmx_prometheus 212 | - ./secrets:/etc/kafka/secrets 213 | environment: 214 | CLUSTER_ID: "0000000000000000000000" 215 | KAFKA_NODE_ID: 3 216 | KAFKA_PROCESS_ROLES: broker 217 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 218 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 219 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:9092,PLAINTEXT_HOST://localhost:39092 220 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 221 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 222 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 223 | KAFKA_MIN_INSYNC_REPLICAS: 2 224 | KAFKA_NUM_PARTITIONS: 4 225 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 226 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 227 | KAFKA_METRIC_REPORTERS: 228 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 229 | CONFLUENT_METRICS_ENABLE: 'false' 230 | 231 | broker-4: 232 | image: confluentinc/cp-kafka:${VERSION} 233 | hostname: broker-4 234 | container_name: kraft-broker-4 235 | stop_grace_period: 60s 236 | healthcheck: 237 | test: nc -z localhost 9092 || exit 1 238 | start_period: 5s 239 | interval: 5s 240 | timeout: 5s 241 | retries: 3 242 | depends_on: 243 | controller-0: 244 | condition: service_healthy 245 | controller-1: 246 | condition: service_healthy 247 | controller-2: 248 | condition: service_healthy 249 | ports: 250 | - '49092:49092' 251 | - '47071:7071' 252 | volumes: 253 | - kraft-bk4:/var/lib/kafka/data 254 | - ./jmx_prometheus:/opt/jmx_prometheus 255 | - ./secrets:/etc/kafka/secrets 256 | environment: 257 | CLUSTER_ID: "0000000000000000000000" 258 | KAFKA_NODE_ID: 4 259 | KAFKA_PROCESS_ROLES: broker 260 | KAFKA_SECURITY_INTER_BROKER_LISTENER_NAME: PLAINTEXT 261 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT 262 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-4:9092,PLAINTEXT_HOST://localhost:49092 263 | KAFKA_CONTROLLER_QUORUM_VOTERS: 10@controller-0:9093,11@controller-1:9093,12@controller-2:9093 264 | KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER 265 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 266 | KAFKA_MIN_INSYNC_REPLICAS: 2 267 | KAFKA_NUM_PARTITIONS: 4 268 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 269 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 500 270 | KAFKA_METRIC_REPORTERS: 271 | KAFKA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/kafka-config.yml -javaagent:/opt/jmx_prometheus/jolokia-jvm.jar=port=7072,host=* 272 | CONFLUENT_METRICS_ENABLE: 'false' 273 | 274 | 275 | kraft-schema-registry: 276 | image: confluentinc/cp-schema-registry:${VERSION} 277 | hostname: schema-registry 278 | container_name: kraft-schema-registry 279 | stop_grace_period: 60s 280 | healthcheck: 281 | test: curl --fail http://localhost:8081/ || exit 1 282 | start_period: 5s 283 | interval: 5s 284 | timeout: 5s 285 | retries: 3 286 | depends_on: 287 | controller-0: 288 | condition: service_healthy 289 | broker-1: 290 | condition: service_healthy 291 | broker-2: 292 | condition: service_healthy 293 | broker-3: 294 | condition: service_healthy 295 | ports: 296 | - '8081:8081' 297 | volumes: 298 | - ./jmx_prometheus:/opt/jmx_prometheus 299 | - ./secrets:/etc/schema-registry/secrets 300 | environment: 301 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 302 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092 303 | KAFKA_METRIC_REPORTERS: 304 | CONFLUENT_METRICS_ENABLE: 'false' 305 | EXTRA_OPTS: -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/sr-config.yml 306 | SCHEMA_REGISTRY_OPTS : -javaagent:/opt/jmx_prometheus/jmx_prometheus_javaagent.jar=7071:/opt/jmx_prometheus/sr-config.yml 307 | --------------------------------------------------------------------------------