├── .gitignore ├── README.md ├── bin ├── config │ └── ksql.properties ├── connect ├── connect-all ├── druid ├── druid_query.sh ├── ksql-restart └── ksql-shell ├── cassandra └── docker-compose.yml ├── connect ├── .env ├── data │ ├── .gitkeep │ ├── datagen │ │ └── .gitkeep │ └── spooldir │ │ ├── .gitkeep │ │ ├── errors │ │ └── .gitkeep │ │ └── finished │ │ └── .gitkeep ├── docker-compose.yml ├── jars │ └── .gitkeep ├── jmx_prometheus │ ├── connect-config.yml │ └── jmx_prometheus_javaagent.jar └── secrets │ ├── aws.properties │ └── mysql.properties ├── dashboards ├── .env ├── docker-compose.yml ├── superset │ ├── Dockerfile │ └── superset_config.py └── turnilo │ ├── Dockerfile │ └── entrypoint.sh ├── demo ├── README.md ├── avro │ └── alias_test.sh ├── cdc │ ├── build.sh │ ├── connectors │ │ ├── datagen-order.json │ │ ├── mysql-backup.json │ │ ├── mysql-cdc.json │ │ ├── mysql-order.json │ │ ├── store.json │ │ ├── store_prep.json │ │ ├── user.json │ │ ├── user_prep.json │ │ └── x.json │ ├── data │ │ ├── datagen │ │ │ ├── order.avro │ │ │ ├── store.avro │ │ │ └── user.avro │ │ └── spooldir │ │ │ ├── stores.csv │ │ │ └── users.csv │ ├── db.sh │ ├── down.sh │ ├── setup.sh │ └── up.sh ├── csv │ ├── build.sh │ ├── down.sh │ └── up.sh ├── druid_late │ ├── NOTES.md │ ├── README.md │ ├── build.sh │ ├── connect.sh │ ├── connect │ │ ├── s3-orders.json │ │ └── s3.json │ ├── down.sh │ ├── druid.sh │ ├── druid │ │ ├── compact.json │ │ ├── orders.json │ │ ├── skus.json │ │ └── skus_batch.json │ ├── mc-config.json │ ├── publisher │ │ ├── build.gradle │ │ ├── gradle.properties │ │ ├── lombok.config │ │ ├── run.sh │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── dev │ │ │ │ └── buesing │ │ │ │ └── ksd │ │ │ │ ├── domain │ │ │ │ ├── PurchaseOrder.java │ │ │ │ └── Sku.java │ │ │ │ ├── parse │ │ │ │ ├── DateTimeConverter.java │ │ │ │ └── DurationConverter.java │ │ │ │ ├── publisher │ │ │ │ ├── Main.java │ │ │ │ ├── Options.java │ │ │ │ └── Producer.java │ │ │ │ └── serde │ │ │ │ ├── JsonDeserializer.java │ │ │ │ ├── JsonSerde.java │ │ │ │ └── JsonSerializer.java │ │ │ └── resources │ │ │ └── logback.xml │ ├── setup.sh │ ├── stop.sh │ ├── superset.sh │ ├── superset │ │ ├── chart.sh │ │ ├── chart_line.json │ │ ├── chart_pivot.json │ │ └── dashboard.sh │ └── up.sh ├── druid_rollup │ ├── INSTALL.md │ ├── README.md │ ├── build.sh │ ├── connectors │ │ ├── order.json │ │ ├── store.json │ │ └── user.json │ ├── data │ │ └── datagen │ │ │ ├── order.avro │ │ │ ├── store.avro │ │ │ └── user.avro │ ├── down.sh │ ├── druid │ │ ├── compact │ │ │ ├── compact.json │ │ │ ├── order_user_reward.json │ │ │ └── order_user_reward_rekeyed.json │ │ ├── compare │ │ │ ├── order_non_rolled.json │ │ │ ├── order_store_zipcode.json │ │ │ ├── order_user_reward.json │ │ │ └── order_user_reward_store_zipcode.json │ │ ├── custom │ │ │ └── order_custom.json │ │ └── sketch │ │ │ ├── order_sketch_hll_4_12.json │ │ │ ├── order_sketch_hll_4_4.json │ │ │ ├── order_sketch_hll_8_12.json │ │ │ ├── order_sketch_none.json │ │ │ ├── order_sketch_theta_16.json │ │ │ ├── order_sketch_theta_16384.json │ │ │ └── order_sketch_theta_32.json │ ├── druid_compact.sh │ ├── druid_compare.sh │ ├── druid_sketch.sh │ ├── ksql │ │ ├── orders.ksql │ │ ├── orders_2.ksql │ │ ├── orders_enriched.ksql │ │ ├── orders_enriched_rekeyed.ksql │ │ ├── stores.ksql │ │ └── users.ksql │ ├── setup.sh │ ├── stop.sh │ └── up.sh ├── eos │ ├── README.md │ ├── build.sh │ ├── connectors │ │ ├── order.json │ │ ├── store.json │ │ ├── store_prep.json │ │ ├── user.json │ │ └── user_prep.json │ ├── data │ │ ├── datagen │ │ │ ├── order.avro │ │ │ ├── store.avro │ │ │ └── user.avro │ │ └── spooldir │ │ │ ├── stores.csv │ │ │ └── users.csv │ ├── down.sh │ ├── druid │ │ ├── order.json │ │ ├── order_metric_showcase.json │ │ ├── order_store_zipcode.json │ │ ├── order_uncommitted.json │ │ ├── order_user_reward.json │ │ ├── order_user_reward_sketch.json │ │ └── order_user_reward_store_zipcode.json │ ├── ksql │ │ ├── orders.ksql │ │ ├── orders_enriched.ksql │ │ ├── stores.ksql │ │ └── users.ksql │ ├── setup.sh │ ├── stop.sh │ └── up.sh ├── jdbc │ └── example.json ├── metrics │ └── druid │ │ └── config.json ├── mongo-cdc │ ├── build.sh │ ├── connectors │ │ ├── mongo-cdc.json │ │ ├── mongo.json │ │ └── mysql-cdc.json │ ├── data │ │ ├── orders.csv │ │ ├── stores.csv │ │ └── users.csv │ ├── down.sh │ ├── generate.sh │ ├── ksql.sh │ ├── ksql │ │ ├── orders.ksql │ │ ├── orders_enriched.ksql │ │ ├── stores.ksql │ │ └── users.ksql │ ├── setup.sh │ └── up.sh ├── mongo │ ├── build.sh │ ├── connectors │ │ ├── mongo.json │ │ └── mysql-cdc.json │ ├── data │ │ ├── orders.csv │ │ ├── stores.csv │ │ └── users.csv │ ├── db.sh │ ├── down.sh │ ├── generate.sh │ ├── ksql.sh │ ├── ksql │ │ ├── orders.ksql │ │ ├── orders_enriched.ksql │ │ ├── stores.ksql │ │ └── users.ksql │ ├── setup.sh │ └── up.sh ├── mysql-postgres-cdc-cassandra │ ├── README.md │ ├── build.sh │ ├── connectors │ │ ├── cassandra.json │ │ ├── cassandra_alt.json │ │ ├── mysql-cdc.json │ │ └── postgres-cdc.json │ ├── data │ │ ├── stores.csv │ │ └── users.csv │ ├── down.sh │ ├── generate.sh │ ├── ksql.sh │ ├── ksql │ │ ├── mysql_orders.ksql │ │ ├── mysql_orders_enriched.ksql │ │ ├── mysql_stores.ksql │ │ ├── mysql_users.ksql │ │ ├── postgres_orders.ksql │ │ ├── postgres_orders_enriched.ksql │ │ ├── postgres_stores.ksql │ │ ├── postgres_users.ksql │ │ └── union.ksql │ ├── setup.sh │ └── up.sh ├── opensky │ ├── README.md │ ├── build.sh │ ├── connectors │ │ └── opensky.json │ ├── down.sh │ ├── druid │ │ ├── flight.json │ │ ├── flight_summary.json │ │ └── flight_summary_2.json │ ├── ksql │ │ ├── config.properties │ │ ├── create.sh │ │ ├── flights.ksql │ │ └── flights_summary.ksql │ ├── setup.sh │ ├── stop.sh │ ├── superset │ │ ├── chart_country.json │ │ ├── create.sh │ │ ├── druid.json │ │ └── login.json │ ├── tmp │ │ └── .gitkeep │ └── up.sh ├── rollups │ └── druid │ │ ├── store_rewards_user.json │ │ └── store_rewards_user_unique.json ├── storage │ ├── down.sh │ ├── f.json │ ├── gcp.json │ ├── m.json │ ├── s3.json │ └── up.sh └── tmp │ └── .gitkeep ├── druid ├── .env ├── docker-compose.yml ├── environment ├── jmx_prometheus │ ├── druid-config.yml │ └── jmx_prometheus_javaagent.jar └── staging │ ├── exports │ └── .gitkeep │ └── scripts │ ├── export_historical.sh │ └── export_rt.sh ├── elasticsearch ├── .env └── docker-compose.yml ├── flink ├── .env └── docker-compose.yml ├── kafka-lb ├── .env ├── docker-compose.yml └── nginx │ ├── Dockerfile │ ├── broker-1.conf │ ├── broker-2.conf │ ├── broker-3.conf │ ├── broker-4.conf │ └── broker_latency.sh ├── kafka ├── .env ├── build.gradle ├── docker-compose.yml └── jmx_prometheus │ ├── jmx_prometheus_javaagent.jar │ ├── jolokia-agent.jar │ ├── kafka-config.yml │ └── sr-config.yml ├── ksqlDB ├── .env ├── docker-compose.yml ├── ext │ └── .gitkeep └── jmx_prometheus │ ├── jmx_prometheus_javaagent.jar │ └── streams-config.yml ├── mongo ├── .env ├── db-init │ └── 00-create-user.js ├── docker-compose.yml ├── restheart │ └── users.yml ├── secret.key └── secret.sh ├── monitoring ├── .env ├── akhq │ └── application.yml ├── cmak │ └── Dockerfile ├── docker-compose.yml ├── grafana │ ├── dashboards.yml │ ├── dashboards │ │ ├── connect │ │ │ ├── kafka_connect_cluster.json │ │ │ ├── kafka_connect_cluster_connector.json │ │ │ ├── kafka_connect_cluster_connector_error.json │ │ │ ├── kafka_connect_cluster_connector_sink.json │ │ │ ├── kafka_connect_cluster_connector_source.json │ │ │ └── kafka_connect_cluster_worker.json │ │ ├── druid │ │ │ ├── druid-ingestion.json │ │ │ ├── druid-overview.json │ │ │ ├── druid-overview2.json │ │ │ └── druid2.json │ │ ├── exporter │ │ │ ├── exporter.json │ │ │ └── kafka_cluster_exporter.json │ │ ├── home.json │ │ ├── jvm.json │ │ ├── kafka │ │ │ ├── 01_kafka_cluster.json │ │ │ ├── 02_kafka_cluster_connections.json │ │ │ ├── 03_kafka_cluster_request.json │ │ │ ├── 04_kafka_cluster_producer.json │ │ │ ├── 05_kafka_cluster_consumer.json │ │ │ └── 06_kafka_cluster_topic.json │ │ ├── schema-registry │ │ │ └── 01_schema_registry.json │ │ └── streams │ │ │ ├── streams_process.json │ │ │ ├── streams_process_blank.json │ │ │ ├── streams_process_e2e.json │ │ │ ├── streams_recordcache.json │ │ │ ├── streams_statestore.json │ │ │ ├── streams_statestore_combined.json │ │ │ ├── streams_statestore_rocksdb_property.json │ │ │ ├── streams_statestore_rocksdb_stats.json │ │ │ ├── streams_task.json │ │ │ ├── streams_task_e2e.json │ │ │ └── streams_thread.json │ ├── druid.yml │ ├── grafana.ini │ └── prometheus.yml ├── kouncil │ └── kouncil.yaml ├── kowl │ └── config.yaml ├── prometheus │ ├── prometheus.yml │ └── targets.json └── static │ ├── consumer_request_metrics.png │ └── producer_request_metrics.png ├── mysql ├── .env ├── cnf │ └── local.cnf ├── docker-compose.yml ├── init.d │ ├── backup.sql │ └── main.sql └── mysql.cnf ├── mysql5 ├── .env ├── cnf │ └── local.cnf ├── docker-compose.yml ├── init.d │ └── main.sql ├── mysql.cnf └── sql │ └── queries.sql ├── network.sh ├── oracle ├── .env ├── README.md ├── docker-compose.yml └── init.d │ └── setup │ └── logminer.sql ├── pinot ├── .env └── docker-compose.yml ├── postgres ├── .env └── docker-compose.yml ├── scripts ├── .zshrc ├── connect_setup.sh ├── container.sh ├── druid_download_extensions.sh ├── druid_dump.sh ├── install_jmxterm.sh ├── ips.sh ├── stats.sh └── volume_sizes.sh ├── settings.gradle └── storage ├── .env ├── README.md ├── docker-compose.yml └── mc-config.json /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | .gradle 4 | grafana.db 5 | build 6 | /*/data/* 7 | 8 | node_modules 9 | package-lock.json 10 | 11 | *.bak 12 | 13 | jmxterm.jar 14 | 15 | /tmp 16 | /ksqlDB/ext/* 17 | /connect/jars/* 18 | 19 | /connect/tmp/* 20 | /druid/storage/* 21 | /druid/staging/* 22 | /oracle/init.d/startup 23 | /oracle/init.d/startup.stage 24 | /flink/jars/* 25 | /flink/tmp/* 26 | /demo/**/tmp/* 27 | 28 | /connect/secrets/opensky.properties 29 | 30 | /gradle 31 | /gradlew 32 | /gradlew.bat 33 | -------------------------------------------------------------------------------- /bin/config/ksql.properties: -------------------------------------------------------------------------------- 1 | auto.offset.reset=earliest 2 | -------------------------------------------------------------------------------- /bin/connect-all: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ $# -lt 2 ]; then 4 | echo "" 5 | echo "Usage: " 6 | echo " $0 {create|pause|resume|delete|status|validate} directory" 7 | echo "" 8 | exit 9 | fi 10 | 11 | BIN_DIR=$(dirname $0) 12 | BASE_DIR=$(dirname $BIN_DIR) 13 | 14 | if [ $# -lt 1 ]; then 15 | echo "usage: $0 {create|pause|resume|delete|status|validate} " 16 | exit 17 | fi 18 | 19 | command=$1 20 | shift 21 | 22 | directory=$1 23 | shift 24 | 25 | declare -a connectors=($(ls $directory/*.json)) 26 | 27 | for i in "${connectors[@]}"; do 28 | ${BIN_DIR}/connect $command $i 29 | done 30 | 31 | -------------------------------------------------------------------------------- /bin/druid: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 {load|unload|task} specification-file [specification-files]" 6 | echo "" 7 | } 8 | 9 | if [ $# -lt 2 ]; then 10 | usage 11 | exit 12 | fi 13 | 14 | OPERATION=$1 15 | shift 16 | 17 | declare -a FILES=( "$@" ) 18 | 19 | URL=http://localhost:48081 20 | 21 | CONTENT_TYPE_HDR='Content-Type:application/json' 22 | 23 | task() { 24 | 25 | DATA=$1 26 | if [ ! -f "$DATA" ]; then 27 | echo "skipping $DATA, file not found." 28 | return 29 | fi 30 | 31 | curl -s -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/indexer/v1/task | jq -r ".task" 32 | #curl -X POST -H "Content-Type:application/json" -d @./druid/compact.json http://localhost:48081/druid/indexer/v1/task 33 | } 34 | 35 | load() { 36 | 37 | DATA=$1 38 | if [ ! -f "$DATA" ]; then 39 | echo "skipping $DATA, file not found." 40 | return 41 | fi 42 | 43 | curl -s -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/indexer/v1/supervisor | jq -r ".id" 44 | } 45 | 46 | unload() { 47 | 48 | DATA=$1 49 | if [ ! -f "$DATA" ]; then 50 | echo "skipping $DATA, file not found." 51 | return 52 | fi 53 | 54 | spec=$(cat $DATA | jq -r ".spec.dataSchema.dataSource | select (.!=null)") 55 | 56 | if [ "${spec}" == "" ]; then 57 | echo "skipping $DATA, unable to extract dataSource from spec." 58 | return 59 | fi 60 | 61 | curl -s -X POST $URL/druid/indexer/v1/supervisor/${spec}/reset 62 | curl -s -X POST $URL/druid/indexer/v1/supervisor/${spec}/shutdown 63 | } 64 | 65 | case "${OPERATION}" in 66 | task) 67 | for i in "${FILES[@]}"; do 68 | task $i 69 | done 70 | ;; 71 | load) 72 | for i in "${FILES[@]}"; do 73 | load $i 74 | done 75 | ;; 76 | unload) 77 | for i in "${FILES[@]}"; do 78 | unload $i 79 | done 80 | ;; 81 | *) 82 | usage 83 | esac 84 | -------------------------------------------------------------------------------- /bin/druid_query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DATA=$1 4 | shift 5 | 6 | URL=http://localhost:48082 7 | #URL=http://localhost:48081 8 | 9 | CONTENT_TYPE_HDR='Content-Type:application/json' 10 | 11 | curl -X POST -H $CONTENT_TYPE_HDR -d @$DATA $URL/druid/v2/sql/ 12 | 13 | -------------------------------------------------------------------------------- /bin/ksql-shell: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 ksql-file" 6 | echo "" 7 | } 8 | 9 | if [ $# -lt 1 ]; then 10 | usage 11 | exit 12 | fi 13 | 14 | DIR=$(dirname $0) 15 | 16 | FILE=$1 17 | shift 18 | 19 | if [ ! -f $FILE ]; then 20 | echo "" 21 | echo "unable to find ksql file." 22 | echo "" 23 | exit 24 | fi 25 | 26 | ksql --config-file $DIR/config/ksql.properties --file=$FILE 27 | 28 | 29 | -------------------------------------------------------------------------------- /cassandra/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: kafka-local 5 | 6 | volumes: 7 | cassandra_data: {} 8 | etc: {} 9 | 10 | services: 11 | cassandra: 12 | image: cassandra:4.0.4 13 | hostname: cassandra 14 | container_name: cassandra 15 | ports: 16 | - "9042:9042" 17 | healthcheck: 18 | test: ["CMD", "cqlsh", "-e", "describe keyspaces" ] 19 | interval: 5s 20 | timeout: 5s 21 | retries: 20 22 | start_period: 20s 23 | volumes: 24 | - cassandra_data:/var/lib/cassandra 25 | - etc:/etc/cassandra 26 | - ./data:/data 27 | environment: 28 | CASSANDRA_CLUSTER_NAME: cassandra 29 | CASSANDRA_DATACENTER: dc_main 30 | CASSANDRA_PASSWORD: cassandra 31 | # CASSANDRA_SEEDS: "cass1,cass2" 32 | # CASSANDRA_RACK: West 33 | # CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch 34 | # CASSANDRA_NUM_TOKENS: 128 35 | -------------------------------------------------------------------------------- /connect/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=connect 2 | -------------------------------------------------------------------------------- /connect/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/data/.gitkeep -------------------------------------------------------------------------------- /connect/data/datagen/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/data/datagen/.gitkeep -------------------------------------------------------------------------------- /connect/data/spooldir/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/data/spooldir/.gitkeep -------------------------------------------------------------------------------- /connect/data/spooldir/errors/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/data/spooldir/errors/.gitkeep -------------------------------------------------------------------------------- /connect/data/spooldir/finished/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/data/spooldir/finished/.gitkeep -------------------------------------------------------------------------------- /connect/jars/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/jars/.gitkeep -------------------------------------------------------------------------------- /connect/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/connect/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /connect/secrets/aws.properties: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID=admin 2 | AWS_SECRET_ACCESS_KEY=miniominio 3 | -------------------------------------------------------------------------------- /connect/secrets/mysql.properties: -------------------------------------------------------------------------------- 1 | CONNECTION_URL=jdbc:mysql://mysql:3306/MAIN 2 | CONNECTION_USER=user 3 | CONNECTION_PASSWORD=userpw 4 | 5 | -------------------------------------------------------------------------------- /dashboards/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=dashboards 2 | -------------------------------------------------------------------------------- /dashboards/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | services: 10 | superset: 11 | build: ./superset 12 | container_name: superset 13 | healthcheck: 14 | test: wget --spider http://localhost:8088/superset/welcome/ || exit 1 15 | start_period: 3s 16 | interval: 3s 17 | timeout: 2s 18 | retries: 3 19 | ports: 20 | - 28088:8088 21 | environment: 22 | WTF_CSRF_ENABLED: "False" 23 | MAPBOX_API_KEY: "${MAPBOX_API_KEY:-}" 24 | 25 | turnilo: 26 | build: ./turnilo 27 | container_name: turnilo 28 | healthcheck: 29 | test: wget --spider http://localhost:9090 || exit 1 30 | start_period: 3s 31 | interval: 3s 32 | timeout: 2s 33 | retries: 3 34 | ports: 35 | - 9091:9090 36 | environment: 37 | DRUID: http://druid-router:8888 38 | 39 | -------------------------------------------------------------------------------- /dashboards/superset/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/superset:latest 2 | 3 | USER root 4 | 5 | #RUN pip install apache-superset[druid] 6 | 7 | RUN pip install pydruid 8 | RUN pip install pydruid[cli] 9 | RUN pip install trino 10 | 11 | #RUN npm i @superset-ui/legacy-preset-chart-kepler 12 | 13 | USER superset 14 | 15 | COPY ./superset_config.py /app/pythonpath/superset_config.py 16 | 17 | RUN export FLASK_APP=superset 18 | RUN superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin 19 | RUN superset db upgrade 20 | RUN superset init 21 | 22 | #RUN superset load_examples 23 | 24 | -------------------------------------------------------------------------------- /dashboards/superset/superset_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | WTF_CSRF_ENABLED = False 5 | 6 | # 7 | # Disable Authorization 8 | # 9 | 10 | #AUTH_ROLE_PUBLIC = 'Public' 11 | #AUTH_USER_REGISTRATION_ROLE = 'Public' 12 | #PUBLIC_ROLE_LIKE_GAMMA = True 13 | 14 | #PUBLIC_ROLE_LIKE 15 | 16 | #Allow for Embedded Javascript within charts that allow it 17 | ENABLE_JAVASCRIPT_CONTROLS = True 18 | 19 | #To Avoid hard-cording of personal MAPBOX_API_KEY into the local image, reference from evironment variable. 20 | MAPBOX_API_KEY = os.getenv('MAPBOX_API_KEY', '') 21 | -------------------------------------------------------------------------------- /dashboards/turnilo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:alpine3.14 2 | 3 | RUN npm install -g turnilo 4 | 5 | COPY ./entrypoint.sh /entrypoint.sh 6 | 7 | ENTRYPOINT /entrypoint.sh 8 | 9 | -------------------------------------------------------------------------------- /dashboards/turnilo/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "${DRUID}" == "" ]; then 4 | echo "environment variable DRUID is not defined." 5 | exit 6 | fi 7 | 8 | echo "Druid URL ${DRUID}" 9 | 10 | turnilo --druid ${DRUID} 11 | 12 | -------------------------------------------------------------------------------- /demo/avro/alias_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | TOPIC=test 4 | 5 | value_v1_schema=$(cat < ../../mysql/data/orders.sql < ../../mysql/data/orders.sql < ../../connect/secrets/aws.properties < /dev/null 47 | 48 | export CP="" 49 | ${cp} 50 | 51 | java -cp \${CP} dev.buesing.ksd.publisher.Main "\$@" 52 | """ 53 | f.setExecutable(true) 54 | } 55 | } 56 | 57 | build.finalizedBy buildRunScript 58 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/gradle.properties: -------------------------------------------------------------------------------- 1 | # 2 | kafka_version=3.1.0 3 | confluent_io_version=7.1.0 4 | # 5 | apache_commons_version=3.12.0 6 | jackson_version=2.12.3 7 | jcommander_version=1.78 8 | lombok_version=1.18.22 9 | logback_version=1.2.10 10 | slf4j_version=1.7.32 11 | # test 12 | junit_version=5.8.2 13 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/lombok.config: -------------------------------------------------------------------------------- 1 | lombok.addLombokGeneratedAnnotation = true 2 | 3 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | gradle assemble > /dev/null 4 | 5 | export CP="" 6 | export CP="${CP}:/Users/buesing/workspaces/nbuesing/kafka-local/demo/druid_late/publisher/build/classes/java/main" 7 | export CP="${CP}:/Users/buesing/workspaces/nbuesing/kafka-local/demo/druid_late/publisher/build/resources/main" 8 | export CP="${CP}:/Users/buesing/.m2/repository/ch/qos/logback/logback-classic/1.2.10/logback-classic-1.2.10.jar" 9 | export CP="${CP}:/Users/buesing/.m2/repository/com/beust/jcommander/1.78/jcommander-1.78.jar" 10 | export CP="${CP}:/Users/buesing/.m2/repository/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.3/jackson-datatype-jsr310-2.12.3.jar" 11 | export CP="${CP}:/Users/buesing/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.12.3/jackson-databind-2.12.3.jar" 12 | export CP="${CP}:/Users/buesing/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.12.3/jackson-core-2.12.3.jar" 13 | export CP="${CP}:/Users/buesing/.m2/repository/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar" 14 | export CP="${CP}:/Users/buesing/.gradle/caches/modules-2/files-2.1/org.apache.kafka/kafka-clients/3.1.0/af924560e38c0a6dbf270bc2d361b2dfab0e03ec/kafka-clients-3.1.0.jar" 15 | export CP="${CP}:/Users/buesing/.m2/repository/org/slf4j/slf4j-api/1.7.32/slf4j-api-1.7.32.jar" 16 | export CP="${CP}:/Users/buesing/.m2/repository/ch/qos/logback/logback-core/1.2.10/logback-core-1.2.10.jar" 17 | export CP="${CP}:/Users/buesing/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.12.3/jackson-annotations-2.12.3.jar" 18 | export CP="${CP}:/Users/buesing/.m2/repository/com/github/luben/zstd-jni/1.5.0-4/zstd-jni-1.5.0-4.jar" 19 | export CP="${CP}:/Users/buesing/.gradle/caches/modules-2/files-2.1/org.lz4/lz4-java/1.8.0/4b986a99445e49ea5fbf5d149c4b63f6ed6c6780/lz4-java-1.8.0.jar" 20 | export CP="${CP}:/Users/buesing/.m2/repository/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar" 21 | 22 | java -cp ${CP} dev.buesing.ksd.publisher.Main "$@" 23 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/src/main/java/dev/buesing/ksd/parse/DateTimeConverter.java: -------------------------------------------------------------------------------- 1 | package dev.buesing.ksd.parse; 2 | 3 | import com.beust.jcommander.ParameterException; 4 | import com.beust.jcommander.converters.BaseConverter; 5 | import java.time.LocalDateTime; 6 | import java.time.format.DateTimeFormatter; 7 | import java.time.format.DateTimeParseException; 8 | 9 | public class DateTimeConverter extends BaseConverter { 10 | 11 | private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 12 | 13 | public DateTimeConverter(String optionName) { 14 | super(optionName); 15 | } 16 | 17 | public LocalDateTime convert(String value) { 18 | try { 19 | return LocalDateTime.parse(value, FORMATTER); 20 | } catch (DateTimeParseException e) { 21 | throw new ParameterException(this.getErrorString(value, String.format("an ISO-8601 formatted date (yyyy-MM-dd HH:mm:ss)"))); 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /demo/druid_late/publisher/src/main/java/dev/buesing/ksd/parse/DurationConverter.java: -------------------------------------------------------------------------------- 1 | package dev.buesing.ksd.parse; 2 | 3 | import com.beust.jcommander.IStringConverter; 4 | import com.beust.jcommander.ParameterException; 5 | import java.time.Duration; 6 | import java.time.format.DateTimeParseException; 7 | 8 | public class DurationConverter implements IStringConverter { 9 | 10 | @Override 11 | public Duration convert(String value) { 12 | try { 13 | return Duration.parse(value); 14 | } catch (DateTimeParseException e) { 15 | throw new ParameterException("Invalid duration"); 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /demo/druid_late/publisher/src/main/java/dev/buesing/ksd/publisher/Main.java: -------------------------------------------------------------------------------- 1 | package dev.buesing.ksd.publisher; 2 | 3 | import com.beust.jcommander.JCommander; 4 | 5 | import java.util.concurrent.ExecutorService; 6 | import java.util.concurrent.Executors; 7 | import java.util.concurrent.Future; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public class Main { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(Producer.class); 14 | 15 | final static Thread.UncaughtExceptionHandler exceptionHandler = (t, e) -> log.error("Uncaught exception in thread '" + t.getName() + "': " + e.getMessage()); 16 | 17 | public static void main(String[] args) { 18 | 19 | final Options options = new Options(); 20 | final JCommander jCommander = JCommander.newBuilder() 21 | .addObject(options) 22 | .build(); 23 | jCommander.parse(args); 24 | 25 | if (options.isHelp()) { 26 | jCommander.usage(); 27 | return; 28 | } 29 | 30 | final ExecutorService executor = Executors.newSingleThreadExecutor(r -> { 31 | final Thread t = Executors.defaultThreadFactory().newThread(r); 32 | t.setDaemon(true); 33 | t.setUncaughtExceptionHandler(exceptionHandler); 34 | return t; 35 | }); 36 | 37 | Future future = executor.submit(() -> { 38 | new Producer(options).start(); 39 | }); 40 | 41 | try { 42 | future.get(); 43 | } catch (Exception e) { 44 | e.printStackTrace(); 45 | } 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/src/main/java/dev/buesing/ksd/serde/JsonSerde.java: -------------------------------------------------------------------------------- 1 | package dev.buesing.ksd.serde; 2 | 3 | import org.apache.kafka.common.serialization.Deserializer; 4 | import org.apache.kafka.common.serialization.Serde; 5 | import org.apache.kafka.common.serialization.Serializer; 6 | 7 | import java.util.Map; 8 | 9 | public class JsonSerde implements Serde { 10 | 11 | public JsonSerde() { 12 | } 13 | 14 | @Override 15 | public void configure(Map configs, boolean isKey) { 16 | } 17 | 18 | @Override 19 | public Serializer serializer() { 20 | return new JsonSerializer<>(); 21 | } 22 | 23 | @Override 24 | public Deserializer deserializer() { 25 | return new JsonDeserializer<>(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/src/main/java/dev/buesing/ksd/serde/JsonSerializer.java: -------------------------------------------------------------------------------- 1 | package dev.buesing.ksd.serde; 2 | 3 | 4 | import com.fasterxml.jackson.databind.DeserializationFeature; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import com.fasterxml.jackson.databind.SerializationFeature; 7 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 8 | import java.util.TimeZone; 9 | import org.apache.kafka.common.errors.SerializationException; 10 | import org.apache.kafka.common.serialization.Serializer; 11 | 12 | import java.util.Map; 13 | 14 | public class JsonSerializer implements Serializer { 15 | 16 | private static final ObjectMapper OBJECT_MAPPER = 17 | new ObjectMapper() 18 | .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) 19 | .setTimeZone(TimeZone.getDefault()) 20 | .configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false) 21 | .registerModule(new JavaTimeModule()); 22 | 23 | // needed by kafka 24 | @SuppressWarnings("unused") 25 | public JsonSerializer() { 26 | } 27 | 28 | @Override 29 | public void configure(Map props, boolean isKey) { 30 | } 31 | 32 | @Override 33 | public byte[] serialize(String topic, T data) { 34 | 35 | if (data == null) 36 | return null; 37 | 38 | try { 39 | return OBJECT_MAPPER.writeValueAsBytes(data); 40 | } catch (Exception e) { 41 | throw new SerializationException("Error serializing JSON message", e); 42 | } 43 | } 44 | 45 | @Override 46 | public void close() { 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /demo/druid_late/publisher/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{0} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /demo/druid_late/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if ! [ -x "$(command -v aws)" ]; then 4 | echo "" 5 | echo "aws is not found, please install and make it available on your path." 6 | echo "" 7 | exit 8 | fi 9 | 10 | 11 | 12 | DIR=$(dirname $0) 13 | 14 | cd $DIR 15 | 16 | BIN=$DIR/../../bin 17 | 18 | DRUID=$BIN/druid 19 | KSQL=$BIN/ksql-shell 20 | 21 | 22 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 23 | 24 | kt --create --replication-factor 3 --partitions 4 --topic orders 25 | kt --create --replication-factor 3 --partitions 4 --topic skus 26 | 27 | MC="docker run -it --rm --network kafka-local --volume $(pwd)/mc-config.json:/root/.mc/config.json minio/mc:latest" 28 | 29 | $MC mb minio/sku 30 | $MC mb minio/order 31 | $MC admin user svcacct add minio admin --access-key druid --secret-key druid_secret 32 | 33 | -------------------------------------------------------------------------------- /demo/druid_late/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | # in case they were started 6 | (cd dashboards; docker compose stop) 7 | 8 | (cd connect; docker compose stop) 9 | (cd druid; docker compose stop) 10 | (cd ksqlDB; docker compose stop) 11 | (cd kafka-lb; docker compose stop) 12 | (cd kafka; docker compose stop) 13 | -------------------------------------------------------------------------------- /demo/druid_late/superset/chart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | API_URL=http://localhost:28088/api/v1 4 | 5 | LOGIN='{ "password": "admin", "provider": "db", "refresh": true, "username": "admin" }' 6 | TOKEN=$(curl -s -X POST -H "Content-Type:application/json" --data "${LOGIN}" ${API_URL}/security/login | jq -r ".access_token") 7 | 8 | echo "TOKEN: ${TOKEN}" 9 | 10 | 11 | curl -s -X GET -H "Authorization:Bearer ${TOKEN}" -H "Content-Type:application/json" -H "Accept:application/json" "${API_URL}/chart/" | jq -r ".result" 12 | 13 | -------------------------------------------------------------------------------- /demo/druid_late/superset/dashboard.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | API_URL=http://localhost:28088/api/v1 4 | 5 | LOGIN='{ "password": "admin", "provider": "db", "refresh": true, "username": "admin" }' 6 | TOKEN=$(curl -s -X POST -H "Content-Type:application/json" --data "${LOGIN}" ${API_URL}/security/login | jq -r ".access_token") 7 | 8 | echo "TOKEN: ${TOKEN}" 9 | 10 | 11 | #curl -s -X GET -H "Authorization:Bearer ${TOKEN}" -H "Content-Type:application/json" -H "Accept:application/json" "${API_URL}/dashboard/" | jq -r ".result" 12 | 13 | curl -s -X POST -H "Authorization:Bearer ${TOKEN}" -H "Content-Type:application/json" -H "Accept:application/json" --data "@./d.json" "${API_URL}/dashboard/" 14 | 15 | -------------------------------------------------------------------------------- /demo/druid_late/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | ./network.sh 6 | 7 | (cd kafka; docker compose up -d $(docker-compose config --services | grep -v schema-registry)) 8 | (cd connect; docker compose up -d) 9 | (cd druid; docker compose up -d) 10 | (cd storage; docker compose up -d minio) 11 | (cd dashboards; docker compose up -d superset --wait) 12 | (cd druid; docker compose up -d --wait) 13 | (cd connect; docker compose up -d --wait) 14 | -------------------------------------------------------------------------------- /demo/druid_rollup/INSTALL.md: -------------------------------------------------------------------------------- 1 | 2 | * `build.sh` 3 | 4 | * Installs `Datagen` connector into Kafka Connect. 5 | 6 | * `up.sh` 7 | 8 | * Starts Kafka Cluster, Kafka Connect, ksqlDB, and Druid. 9 | 10 | * `setup.sh` 11 | 12 | * Verifies connect RESTful API is up and available, aborts immediately if it is not. 13 | 14 | * Copies the data needed by the `Datagen` connector to where it is accessible by the connector from the distributed connect cluster. 15 | 16 | * Creates the Kafka topics `Datagen` 17 | 18 | * Starts the 3 `Dategen` connectors: `user`, `store`, and `order`. 19 | 20 | * Creates ksql for ingesting `users`, `stores`, and `orders`. 21 | 22 | * Create the enriched order that combines and `order` with the store and user, key the topic by `ORDER_ID`. Make sure the ORDER_ID remains in the value, 23 | since Druid cannot pull data from the key. 24 | 25 | * Create another ksql stream to rekey the enriched order to where the topic is keyed by `USER_REWARD`. Make sure the USER_REWARD remains in the value, 26 | since Druid cannot pull data from the key. 27 | 28 | * `down.sh` 29 | 30 | * shuts down all the containers and remove volumes. 31 | -------------------------------------------------------------------------------- /demo/druid_rollup/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd "$(dirname $0)" || exit 4 | 5 | touch ../tmp/connect-distributed.properties 6 | 7 | if [ ! -d "../../connect/jars/confluentinc-kafka-connect-datagen" ]; then 8 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt confluentinc/kafka-connect-datagen:0.5.0 9 | fi 10 | 11 | rm -f ../tmp/connect-distributed.properties 12 | 13 | cp -r data/* ../../connect/data 14 | -------------------------------------------------------------------------------- /demo/druid_rollup/connectors/order.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", 3 | "kafka.topic": "orders", 4 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 5 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 6 | "value.converter.schemas.enable": "false", 7 | "max.interval": 2, 8 | "iterations": 1800000, 9 | "tasks.max": "1", 10 | 11 | "schema.filename": "/data/datagen/order.avro", 12 | 13 | "transforms" : "tokey,cast,extract,insert_ts,InsertField", 14 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 15 | "transforms.tokey.fields" : "order_id", 16 | 17 | "transforms.cast.type" : "org.apache.kafka.connect.transforms.Cast$Key", 18 | "transforms.cast.spec" : "order_id:string", 19 | 20 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 21 | "transforms.extract.field" : "order_id", 22 | 23 | "transforms.insert_ts.type" : "org.apache.kafka.connect.transforms.InsertField$Value", 24 | "transforms.insert_ts.timestamp.field": "TS", 25 | 26 | "transforms.InsertField.type": "org.apache.kafka.connect.transforms.InsertField$Value", 27 | "transforms.InsertField.static.field": "tN", 28 | "transforms.InsertField.static.value": "roTechClockDetailsUpdateEvent", 29 | 30 | 31 | "_producer.override.request.timeout.ms" : 5000 32 | } 33 | -------------------------------------------------------------------------------- /demo/druid_rollup/connectors/store.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", 3 | 4 | "kafka.topic": "stores", 5 | 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 8 | "value.converter.schemas.enable": "false", 9 | 10 | "max.interval": 0, 11 | "iterations": 10000, 12 | "tasks.max": "1", 13 | 14 | "schema.filename": "/data/datagen/store.avro", 15 | 16 | "transforms" : "tokey,cast,extract", 17 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 18 | "transforms.tokey.fields" : "store_id", 19 | 20 | "transforms.cast.type" : "org.apache.kafka.connect.transforms.Cast$Key", 21 | "transforms.cast.spec" : "store_id:string", 22 | 23 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 24 | "transforms.extract.field" : "store_id" 25 | 26 | } 27 | -------------------------------------------------------------------------------- /demo/druid_rollup/connectors/user.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", 3 | 4 | "kafka.topic": "users", 5 | 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 8 | "value.converter.schemas.enable": "false", 9 | 10 | "max.interval": 0, 11 | "iterations": 100000, 12 | "tasks.max": "1", 13 | 14 | "schema.filename": "/data/datagen/user.avro", 15 | 16 | "transforms" : "tokey,cast,extract", 17 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 18 | "transforms.tokey.fields" : "user_id", 19 | 20 | "transforms.cast.type" : "org.apache.kafka.connect.transforms.Cast$Key", 21 | "transforms.cast.spec" : "user_id:string", 22 | 23 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 24 | "transforms.extract.field" : "user_id" 25 | 26 | } 27 | -------------------------------------------------------------------------------- /demo/druid_rollup/data/datagen/order.avro: -------------------------------------------------------------------------------- 1 | { 2 | "namespace":"ksql", 3 | "name":"orders", 4 | "type":"record", 5 | "fields":[ 6 | { 7 | "name":"order_id", 8 | "type":{ 9 | "type":"int", 10 | "arg.properties":{ 11 | "iteration":{ 12 | "start":0 13 | } 14 | } 15 | } 16 | }, 17 | { 18 | "name":"code", 19 | "type":{ 20 | "type":"string", 21 | "arg.properties":{ 22 | "regex":"[A-Z]{0,4}" 23 | } 24 | } 25 | }, 26 | { 27 | "name":"store_id", 28 | "type":{ 29 | "type":"string", 30 | "arg.properties":{ 31 | "regex":"[0-9]{0,4}" 32 | } 33 | } 34 | }, 35 | { 36 | "name":"user_id", 37 | "type":{ 38 | "type":"string", 39 | "arg.properties":{ 40 | "regex":"[0-9]{0,5}" 41 | } 42 | } 43 | }, 44 | { 45 | "name":"quantity", 46 | "type":{ 47 | "type":"int", 48 | "arg.properties":{ 49 | "range":{ 50 | "min":1, 51 | "max":10 52 | } 53 | } 54 | } 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /demo/druid_rollup/data/datagen/store.avro: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "ksql", 3 | "name": "store", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "store_id", 8 | "type": { 9 | "type": "int", 10 | "arg.properties": { 11 | "iteration": { 12 | "start": 0 13 | } 14 | } 15 | } 16 | }, 17 | { 18 | "name":"zipcode", 19 | "type":{ 20 | "type":"string", 21 | "arg.properties":{ 22 | "regex":"55[0-2][0-9]{2}" 23 | } 24 | } 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /demo/druid_rollup/data/datagen/user.avro: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "ksql", 3 | "name": "user", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "user_id", 8 | "type": { 9 | "type": "int", 10 | "arg.properties": { 11 | "iteration": { 12 | "start": 0 13 | } 14 | } 15 | } 16 | }, 17 | { 18 | "name": "reward", 19 | "type": { 20 | "type": "enum", 21 | "name": "Reward", 22 | "symbols": [ 23 | "NONE", 24 | "BRONZE", 25 | "SILVER", 26 | "GOLD", 27 | "PLATINUM", 28 | "DIAMOND" 29 | ], 30 | "arg.properties": { 31 | "options": [ 32 | "NONE", 33 | "BRONZE", 34 | "SILVER", 35 | "GOLD", 36 | "PLATINUM", 37 | "DIAMOND" 38 | ] 39 | } 40 | } 41 | }, 42 | { 43 | "name": "phone_number", 44 | "type": { 45 | "type": "string", 46 | "arg.properties": { 47 | "regex":"(218|320|507|612|651|763|952)-[1-9][0-9]{2}-[0-9]{4}" 48 | } 49 | } 50 | } 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /demo/druid_rollup/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | # in case they were started 6 | (cd dashboards; docker compose down -v) 7 | (cd monitoring; docker compose down -v) 8 | 9 | (cd connect; docker compose down -v) 10 | (cd druid; docker compose down -v) 11 | (cd ksqlDB; docker compose down -v) 12 | (cd kafka-lb; docker compose down -v) 13 | (cd kafka; docker compose down -v) 14 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compact/compact.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "compact", 3 | "dataSource" : "order_user_reward", 4 | "interval": "2021-01-01/2022-01-01", 5 | "tuningConfig" : { 6 | "type" : "index_parallel", 7 | "maxRowsPerSegment" : 500000, 8 | "maxRowsInMemory" : 25000 9 | } 10 | } -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compact/order_user_reward.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 4 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_user_reward", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "fifteen_minute", 50 | "queryGranularity": "minute", 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [] 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compact/order_user_reward_rekeyed.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED_REKEYED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 4 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_user_reward_topic_key_order_id", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "fifteen_minute", 50 | "queryGranularity": "minute", 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [] 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compare/order_non_rolled.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD", 34 | "STORE_ZIPCODE", 35 | { 36 | "type": "long", 37 | "name" : "QUANTITY", 38 | "field": "QUANTITY" 39 | } 40 | ], 41 | "dimensionExclusions": [ 42 | ] 43 | }, 44 | "metricsSpec": [ 45 | { 46 | "type": "count", 47 | "name": "COUNT" 48 | } 49 | ], 50 | "granularitySpec": { 51 | "type": "uniform", 52 | "segmentGranularity": "fifteen_minute", 53 | "queryGranularity": "minute", 54 | "rollup": false, 55 | "intervals": null 56 | }, 57 | "transformSpec": { 58 | "filter": null, 59 | "transforms": [ 60 | ] 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compare/order_store_zipcode.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_store_zipcode", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "STORE_ZIPCODE" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "fifteen_minute", 50 | "queryGranularity": "minute", 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [] 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compare/order_user_reward.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_user_reward", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "fifteen_minute", 50 | "queryGranularity": "minute", 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [] 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/compare/order_user_reward_store_zipcode.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_user_reward_store_zipcode", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD", 34 | "STORE_ZIPCODE" 35 | ] 36 | }, 37 | "metricsSpec": [ 38 | { 39 | "type": "count", 40 | "name": "COUNT" 41 | }, 42 | { 43 | "type": "longSum", 44 | "name": "QUANTITY", 45 | "fieldName": "QUANTITY" 46 | } 47 | ], 48 | "granularitySpec": { 49 | "type": "uniform", 50 | "segmentGranularity": "fifteen_minute", 51 | "queryGranularity": "minute", 52 | "rollup": true, 53 | "intervals": null 54 | }, 55 | "transformSpec": { 56 | "transforms": [] 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/custom/order_custom.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_custom_2", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "fifteen_minute", 50 | "queryGranularity": null, 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [ 56 | { 57 | "type": "expression", 58 | "name": "USER_PHONE_NUMBER", 59 | "expression" : "substring(USER_PHONE_NUMBER, 0, 3)" 60 | }, 61 | { 62 | "type": "expression", 63 | "name": "__time", 64 | "expression": "if(timestamp_extract(\"__time\", 'MINUTE') % 2, \"__time\" / 1000 * 1000, \"__time\" / 5000 * 5000)" 65 | } 66 | ] 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_hll_4_12.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_hll_4_12", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "STORE_ZIPCODE", 34 | "USER_REWARD" 35 | ], 36 | "dimensionExclusions" : [ 37 | ] 38 | }, 39 | "metricsSpec": [ 40 | { 41 | "type": "count", 42 | "name": "COUNT" 43 | }, 44 | { 45 | "type" : "HLLSketchBuild", 46 | "name" : "UNIQUE_CODES", 47 | "fieldName" : "CODE", 48 | "lgK" : 12, 49 | "tgtHllType" : "HLL_4" 50 | } 51 | ], 52 | "granularitySpec": { 53 | "type": "uniform", 54 | "segmentGranularity": "hour", 55 | "queryGranularity": {"type": "duration", "duration": 300000}, 56 | "rollup": true, 57 | "intervals": null 58 | }, 59 | "transformSpec": { 60 | "transforms": [ 61 | ] 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_hll_4_4.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_hll_4_4", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "STORE_ZIPCODE", 34 | "USER_REWARD" 35 | ], 36 | "dimensionExclusions" : [ 37 | ] 38 | }, 39 | "metricsSpec": [ 40 | { 41 | "type": "count", 42 | "name": "COUNT" 43 | }, 44 | { 45 | "type" : "HLLSketchBuild", 46 | "name" : "UNIQUE_CODES", 47 | "fieldName" : "CODE", 48 | "lgK" : 4, 49 | "tgtHllType" : "HLL_4" 50 | } 51 | ], 52 | "granularitySpec": { 53 | "type": "uniform", 54 | "segmentGranularity": "hour", 55 | "queryGranularity": {"type": "duration", "duration": 300000}, 56 | "rollup": true, 57 | "intervals": null 58 | }, 59 | "transformSpec": { 60 | "transforms": [ 61 | ] 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_hll_8_12.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_hll_8_12", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "STORE_ZIPCODE", 34 | "USER_REWARD" 35 | ], 36 | "dimensionExclusions" : [ 37 | ] 38 | }, 39 | "metricsSpec": [ 40 | { 41 | "type": "count", 42 | "name": "COUNT" 43 | }, 44 | { 45 | "type" : "HLLSketchBuild", 46 | "name" : "UNIQUE_CODES", 47 | "fieldName" : "CODE", 48 | "lgK" : 12, 49 | "tgtHllType" : "HLL_8" 50 | } 51 | ], 52 | "granularitySpec": { 53 | "type": "uniform", 54 | "segmentGranularity": "hour", 55 | "queryGranularity": {"type": "duration", "duration": 300000}, 56 | "rollup": true, 57 | "intervals": null 58 | }, 59 | "transformSpec": { 60 | "transforms": [ 61 | ] 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_none.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_none", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "STORE_ZIPCODE", 34 | "USER_REWARD", 35 | "CODE" 36 | ] 37 | }, 38 | "metricsSpec": [ 39 | { 40 | "type": "count", 41 | "name": "COUNT" 42 | } 43 | ], 44 | "granularitySpec": { 45 | "type": "uniform", 46 | "segmentGranularity": "hour", 47 | "queryGranularity": {"type": "duration", "duration": 300000}, 48 | "rollup": true, 49 | "intervals": null 50 | }, 51 | "transformSpec": { 52 | "transforms": [ 53 | ] 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_theta_16.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_theta_16", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD", 34 | "STORE_ZIPCODE" 35 | ], 36 | "dimensionExclusions" : [ 37 | ] 38 | }, 39 | "metricsSpec": [ 40 | { 41 | "type": "count", 42 | "name": "COUNT" 43 | }, 44 | { 45 | "type" : "thetaSketch", 46 | "name" : "UNIQUE_CODES", 47 | "fieldName" : "CODE", 48 | "size" : "16" 49 | } 50 | ], 51 | "granularitySpec": { 52 | "type": "uniform", 53 | "segmentGranularity": "hour", 54 | "queryGranularity": {"type": "duration", "duration": 300000}, 55 | "rollup": true, 56 | "intervals": null 57 | }, 58 | "transformSpec": { 59 | "transforms": [ 60 | ] 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_theta_16384.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_theta_16384", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD", 34 | "STORE_ZIPCODE" 35 | ], 36 | "dimensionExclusions" : [ 37 | ] 38 | }, 39 | "metricsSpec": [ 40 | { 41 | "type": "count", 42 | "name": "COUNT" 43 | }, 44 | { 45 | "type" : "thetaSketch", 46 | "name" : "UNIQUE_CODES", 47 | "fieldName" : "CODE", 48 | "size" : "16384" 49 | } 50 | ], 51 | "granularitySpec": { 52 | "type": "uniform", 53 | "segmentGranularity": "hour", 54 | "queryGranularity": {"type": "duration", "duration": 300000}, 55 | "rollup": true, 56 | "intervals": null 57 | }, 58 | "transformSpec": { 59 | "transforms": [ 60 | ] 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid/sketch/order_sketch_theta_32.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 100000, 22 | "maxRowsPerSegment": 500000, 23 | "maxTotalRows": 2000000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_sketch_theta_32", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD", 34 | "STORE_ZIPCODE" 35 | ], 36 | "dimensionExclusions" : [ 37 | ] 38 | }, 39 | "metricsSpec": [ 40 | { 41 | "type": "count", 42 | "name": "COUNT" 43 | }, 44 | { 45 | "type" : "thetaSketch", 46 | "name" : "UNIQUE_CODES", 47 | "fieldName" : "CODE", 48 | "size" : "32" 49 | } 50 | ], 51 | "granularitySpec": { 52 | "type": "uniform", 53 | "segmentGranularity": "hour", 54 | "queryGranularity": {"type": "duration", "duration": 300000}, 55 | "rollup": true, 56 | "intervals": null 57 | }, 58 | "transformSpec": { 59 | "transforms": [ 60 | ] 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid_compact.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DRUID=$(dirname $0)/../../bin/druid 4 | 5 | $DRUID load ./druid/compact/order_user_reward.json 6 | $DRUID load ./druid/compact/order_user_reward_rekeyed.json 7 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid_compare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DRUID=$(dirname $0)/../../bin/druid 4 | 5 | $DRUID load ./druid/compare/order_non_rolled.json 6 | $DRUID load ./druid/compare/order_store_zipcode.json 7 | $DRUID load ./druid/compare/order_user_reward.json 8 | $DRUID load ./druid/compare/order_user_reward_store_zipcode.json 9 | -------------------------------------------------------------------------------- /demo/druid_rollup/druid_sketch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DRUID=$(dirname $0)/../../bin/druid 4 | 5 | 6 | COMMAND=load 7 | 8 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_none.json 9 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_theta_16.json 10 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_theta_32.json 11 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_theta_16384.json 12 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_hll_4_4.json 13 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_hll_4_12.json 14 | $DRUID ${COMMAND} ./druid/sketch/order_sketch_hll_8_12.json 15 | 16 | -------------------------------------------------------------------------------- /demo/druid_rollup/ksql/orders.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS ( 2 | order_id bigint, 3 | code varchar, 4 | user_id varchar, 5 | store_id varchar, 6 | quantity bigint, 7 | address struct 8 | ) with (kafka_topic='orders', value_format='json'); 9 | 10 | describe ORDERS extended; 11 | -------------------------------------------------------------------------------- /demo/druid_rollup/ksql/orders_2.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS_ENRICHED 2 | as 3 | select 4 | cast(O.ORDER_ID as string) as ORDER_KEY, 5 | O.ROWTIME as ORDER_TS, 6 | AS_VALUE(O.ORDER_ID) as ORDER_ID, 7 | O.USER_ID as USER_ID, 8 | U.REWARD as USER_REWARD, 9 | U.AREA_CODE as USER_AREA_CODE, 10 | O.STORE_ID as STORE_ID, 11 | S.ZIPCODE as STORE_ZIPCODE, 12 | O.QUANTITY as QUANTITY 13 | from ORDERS O 14 | join USERS U on U.USER_ID = O.USER_ID 15 | join STORES S on S.STORE_ID = O.STORE_ID 16 | partition by cast(O.ORDER_ID as string); 17 | 18 | describe ORDERS_ENRICHED extended; 19 | -------------------------------------------------------------------------------- /demo/druid_rollup/ksql/orders_enriched.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS_ENRICHED 2 | as 3 | select 4 | cast(O.ORDER_ID as string) as ORDER_KEY, 5 | O.ROWTIME as ORDER_TS, 6 | AS_VALUE(O.ORDER_ID) as ORDER_ID, 7 | O.CODE as CODE, 8 | O.USER_ID as USER_ID, 9 | U.REWARD as USER_REWARD, 10 | U.PHONE_NUMBER as USER_PHONE_NUMBER, 11 | O.STORE_ID as STORE_ID, 12 | S.ZIPCODE as STORE_ZIPCODE, 13 | O.QUANTITY as QUANTITY 14 | from ORDERS O 15 | join USERS U on U.USER_ID = O.USER_ID 16 | join STORES S on S.STORE_ID = O.STORE_ID 17 | partition by cast(O.ORDER_ID as string); 18 | 19 | describe ORDERS_ENRICHED extended; 20 | -------------------------------------------------------------------------------- /demo/druid_rollup/ksql/orders_enriched_rekeyed.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS_ENRICHED_REKEYED 2 | as 3 | select 4 | USER_REWARD as PARTITION_KEY, 5 | ORDER_TS as ORDER_TS, 6 | ORDER_ID as ORDER_ID, 7 | CODE, 8 | USER_ID, 9 | AS_VALUE(USER_REWARD) as USER_REWARD, 10 | USER_PHONE_NUMBER, 11 | STORE_ID, 12 | STORE_ZIPCODE, 13 | QUANTITY 14 | from ORDERS_ENRICHED O 15 | partition by USER_REWARD; 16 | 17 | describe ORDERS_ENRICHED_REKEYED extended; 18 | -------------------------------------------------------------------------------- /demo/druid_rollup/ksql/stores.ksql: -------------------------------------------------------------------------------- 1 | create or replace table STORES ( 2 | store_id varchar primary key, 3 | zipcode varchar 4 | ) with (kafka_topic='stores', value_format='json'); 5 | 6 | describe STORES extended; 7 | -------------------------------------------------------------------------------- /demo/druid_rollup/ksql/users.ksql: -------------------------------------------------------------------------------- 1 | create or replace table USERS ( 2 | user_id varchar primary key, 3 | reward varchar, 4 | phone_number varchar 5 | ) with (kafka_topic='users', value_format='json'); 6 | 7 | describe USERS extended; 8 | -------------------------------------------------------------------------------- /demo/druid_rollup/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR=$(dirname $0) 4 | 5 | cd $DIR 6 | 7 | BIN=$DIR/../../bin 8 | 9 | CONNECT=$BIN/connect 10 | DRUID=$BIN/druid 11 | KSQL=$BIN/ksql-shell 12 | 13 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 14 | 15 | $CONNECT available 16 | [ $? -ne 0 ] && echo "connector RESTful API is not yet available, aborting script. wait until connector is ready to run this script." && exit 1 17 | 18 | # also done in build.sh 19 | cp -r data/* ../../connect/data 20 | 21 | kt --create --replication-factor 3 --partitions 4 --topic users 22 | kt --create --replication-factor 3 --partitions 4 --topic stores 23 | kt --create --replication-factor 3 --partitions 4 --topic orders 24 | kt --create --replication-factor 3 --partitions 4 --topic ORDERS_ENRICHED 25 | 26 | $CONNECT create ./connectors/store.json 27 | $CONNECT create ./connectors/user.json 28 | $CONNECT create ./connectors/order.json 29 | 30 | $KSQL ./ksql/users.ksql 31 | $KSQL ./ksql/stores.ksql 32 | $KSQL ./ksql/orders.ksql 33 | 34 | echo "sleeping 2 seconds to make sure the tables for users and stores are hydrated before orders enrichment stream is created" 35 | sleep 2 36 | $KSQL ./ksql/orders_enriched.ksql 37 | $KSQL ./ksql/orders_enriched_rekeyed.ksql 38 | -------------------------------------------------------------------------------- /demo/druid_rollup/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | # in case they were started 6 | (cd dashboards; docker compose stop) 7 | 8 | (cd connect; docker compose stop) 9 | (cd druid; docker compose stop) 10 | (cd ksqlDB; docker compose stop) 11 | (cd kafka-lb; docker compose stop) 12 | (cd kafka; docker compose stop) 13 | -------------------------------------------------------------------------------- /demo/druid_rollup/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | ./network.sh 6 | 7 | (cd kafka; docker compose up -d $(docker-compose config --services | grep -v schema-registry)) 8 | (cd connect; docker compose up -d connect-1) 9 | (cd druid; docker compose up -d) 10 | sleep 2 11 | (cd ksqlDB; docker compose up -d) 12 | 13 | #kt --topic _confluent-ksql-default__command_topic --describe 14 | # --partitions 1 --replication-factor 1 15 | # min.insync.replicas=1,cleanup.policy=delete,retention.ms=-1,unclean.leader.election.enable=false 16 | 17 | -------------------------------------------------------------------------------- /demo/eos/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | 5 | touch ../tmp/connect-distributed.properties 6 | 7 | if [ ! -d "../../connect/jars/confluentinc-kafka-connect-datagen" ]; then 8 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt confluentinc/kafka-connect-datagen:0.5.0 9 | fi 10 | 11 | #if [ ! -d "../../connect/jars/streamthoughts-kafka-connect-file-pulse" ]; then 12 | # confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt streamthoughts/kafka-connect-file-pulse:2.3.0 13 | #fi 14 | 15 | if [ ! -d "../../connect/jars/jcustenborder-kafka-connect-spooldir" ]; then 16 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt jcustenborder/kafka-connect-spooldir:2.0.62 17 | fi 18 | 19 | rm -f ../tmp/connect-distributed.properties 20 | 21 | 22 | 23 | cp -r data/* ../../connect/data 24 | -------------------------------------------------------------------------------- /demo/eos/connectors/order.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", 3 | "kafka.topic": "orders", 4 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 5 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 6 | "value.converter.schemas.enable": "false", 7 | "max.interval": 2, 8 | "iterations": 20000000, 9 | "tasks.max": "1", 10 | 11 | "schema.filename": "/data/datagen/order.avro", 12 | 13 | "transforms" : "tokey,cast,extract,insert_ts", 14 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 15 | "transforms.tokey.fields" : "order_id", 16 | 17 | "transforms.cast.type" : "org.apache.kafka.connect.transforms.Cast$Key", 18 | "transforms.cast.spec" : "order_id:string", 19 | 20 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 21 | "transforms.extract.field" : "order_id", 22 | 23 | "transforms.insert_ts.type" : "org.apache.kafka.connect.transforms.InsertField$Value", 24 | "transforms.insert_ts.timestamp.field": "TS", 25 | 26 | 27 | "_producer.override.request.timeout.ms" : 5000 28 | } 29 | -------------------------------------------------------------------------------- /demo/eos/connectors/store.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector", 3 | 4 | "tasks.max": "1", 5 | 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 8 | "value.converter.schemas.enable": "false", 9 | 10 | "topic": "stores", 11 | 12 | "csv.first.row.as.header": "true", 13 | "schema.generation.enabled": "false", 14 | 15 | "key.schema" : "{\"name\":\"store_key\",\"type\":\"STRUCT\",\"isOptional\":false,\"fieldSchemas\":{\"store_id\":{\"type\":\"STRING\",\"isOptional\":false}}}", 16 | "value.schema" : "{\"name\":\"store\",\"type\":\"STRUCT\",\"isOptional\":false,\"fieldSchemas\":{\"store_id\":{\"type\":\"STRING\",\"isOptional\":false},\"zipcode\":{\"type\":\"STRING\",\"isOptional\":false}}}", 17 | 18 | "input.path": "/data/spooldir", 19 | "input.file.pattern": "stores.csv", 20 | "error.path": "/data/spooldir/errors", 21 | "finished.path": "/data/spooldir/finished", 22 | 23 | "transforms" : "tokey,extract", 24 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 25 | "transforms.tokey.fields" : "store_id", 26 | 27 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 28 | "transforms.extract.field" : "store_id", 29 | 30 | "halt.on.error": "true" 31 | 32 | } 33 | 34 | -------------------------------------------------------------------------------- /demo/eos/connectors/store_prep.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", 3 | 4 | "kafka.topic": "stores_prep", 5 | 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 8 | "value.converter.schemas.enable": "false", 9 | 10 | "max.interval": 0, 11 | "iterations": 100, 12 | "tasks.max": "1", 13 | 14 | "schema.filename": "/data/datagen/store.avro", 15 | 16 | "transforms" : "tokey,cast,extract", 17 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 18 | "transforms.tokey.fields" : "store_id", 19 | 20 | "transforms.cast.type" : "org.apache.kafka.connect.transforms.Cast$Key", 21 | "transforms.cast.spec" : "store_id:string", 22 | 23 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 24 | "transforms.extract.field" : "store_id" 25 | 26 | } 27 | -------------------------------------------------------------------------------- /demo/eos/connectors/user.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector", 3 | 4 | "tasks.max": "1", 5 | 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 8 | "value.converter.schemas.enable": "false", 9 | 10 | "topic": "users", 11 | 12 | "csv.first.row.as.header": "true", 13 | "schema.generation.enabled": "false", 14 | 15 | "key.schema" : "{\"name\":\"user_key\",\"type\":\"STRUCT\",\"isOptional\":false,\"fieldSchemas\":{\"user_id\":{\"type\":\"STRING\",\"isOptional\":false}}}", 16 | "value.schema" : "{\"name\":\"user\",\"type\":\"STRUCT\",\"isOptional\":false,\"fieldSchemas\":{\"user_id\":{\"type\":\"STRING\",\"isOptional\":false},\"reward\":{\"type\":\"STRING\",\"isOptional\":false},\"area_code\":{\"type\":\"STRING\",\"isOptional\":false}}}", 17 | 18 | "input.path": "/data/spooldir", 19 | "input.file.pattern": "users.csv", 20 | "error.path": "/data/spooldir/errors", 21 | "finished.path": "/data/spooldir/finished", 22 | 23 | "transforms" : "tokey,extract", 24 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 25 | "transforms.tokey.fields" : "user_id", 26 | 27 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 28 | "transforms.extract.field" : "user_id", 29 | 30 | "halt.on.error": "false" 31 | 32 | } 33 | 34 | -------------------------------------------------------------------------------- /demo/eos/connectors/user_prep.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", 3 | 4 | "kafka.topic": "users_prep", 5 | 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 8 | "value.converter.schemas.enable": "false", 9 | 10 | "max.interval": 0, 11 | "iterations": 1000, 12 | "tasks.max": "1", 13 | 14 | "schema.filename": "/data/datagen/user.avro", 15 | 16 | "transforms" : "tokey,cast,extract", 17 | "transforms.tokey.type" : "org.apache.kafka.connect.transforms.ValueToKey", 18 | "transforms.tokey.fields" : "user_id", 19 | 20 | "transforms.cast.type" : "org.apache.kafka.connect.transforms.Cast$Key", 21 | "transforms.cast.spec" : "user_id:string", 22 | 23 | "transforms.extract.type" : "org.apache.kafka.connect.transforms.ExtractField$Key", 24 | "transforms.extract.field" : "user_id" 25 | 26 | } 27 | -------------------------------------------------------------------------------- /demo/eos/data/datagen/store.avro: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "ksql", 3 | "name": "store", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "store_id", 8 | "type": { 9 | "type": "int", 10 | "arg.properties": { 11 | "iteration": { 12 | "start": 0 13 | } 14 | } 15 | } 16 | }, 17 | { 18 | "name":"zipcode", 19 | "type":{ 20 | "type":"long", 21 | "arg.properties": { 22 | "options": [ 23 | 55101, 24 | 55102, 25 | 55103, 26 | 55104, 27 | 55105, 28 | 55106, 29 | 55107, 30 | 55108, 31 | 55111, 32 | 55114, 33 | 55116, 34 | 55117, 35 | 55119, 36 | 55130, 37 | 55401, 38 | 55402, 39 | 55403, 40 | 55404, 41 | 55405, 42 | 55406, 43 | 55407, 44 | 55408, 45 | 55409, 46 | 55410, 47 | 55411, 48 | 55412, 49 | 55413, 50 | 55414, 51 | 55415, 52 | 55417, 53 | 55418, 54 | 55419, 55 | 55420 56 | ] 57 | } 58 | } 59 | } 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /demo/eos/data/datagen/user.avro: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "ksql", 3 | "name": "user", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "user_id", 8 | "type": { 9 | "type": "int", 10 | "arg.properties": { 11 | "iteration": { 12 | "start": 0 13 | } 14 | } 15 | } 16 | }, 17 | { 18 | "name": "reward", 19 | "type": { 20 | "type": "enum", 21 | "name": "Reward", 22 | "symbols": [ 23 | "NONE", 24 | "SILVER", 25 | "GOLD", 26 | "PLATINUM" 27 | ], 28 | "arg.properties": { 29 | "options": [ 30 | "NONE", 31 | "SILVER", 32 | "GOLD", 33 | "PLATINUM" 34 | ] 35 | } 36 | } 37 | }, 38 | { 39 | "name": "area_code", 40 | "type": { 41 | "type": "string", 42 | "arg.properties": { 43 | "options": [ 44 | "218", 45 | "320", 46 | "507", 47 | "612", 48 | "651", 49 | "763", 50 | "952", 51 | "262", 52 | "414", 53 | "534", 54 | "608", 55 | "715", 56 | "920" 57 | ] 58 | } 59 | } 60 | } 61 | ] 62 | } 63 | -------------------------------------------------------------------------------- /demo/eos/data/spooldir/stores.csv: -------------------------------------------------------------------------------- 1 | store_id,zipcode 2 | "00","55418" 3 | "01","55411" 4 | "02","55410" 5 | "03","55409" 6 | "04","55405" 7 | "05","55101" 8 | "06","55420" 9 | "07","55412" 10 | "08","55101" 11 | "09","55402" 12 | "10","55404" 13 | "11","55418" 14 | "12","55405" 15 | "13","55410" 16 | "14","55404" 17 | "15","55411" 18 | "16","55417" 19 | "17","55414" 20 | "18","55404" 21 | "19","55418" 22 | "20","55104" 23 | "21","55409" 24 | "22","55418" 25 | "23","55104" 26 | "24","55106" 27 | "25","55406" 28 | "26","55102" 29 | "27","55107" 30 | "28","55404" 31 | "29","55117" 32 | "30","55411" 33 | "31","55420" 34 | "32","55104" 35 | "33","55406" 36 | "34","55403" 37 | "35","55406" 38 | "36","55402" 39 | "37","55406" 40 | "38","55415" 41 | "39","55420" 42 | "40","55413" 43 | "41","55407" 44 | "42","55108" 45 | "43","55412" 46 | "44","55412" 47 | "45","55103" 48 | "46","55403" 49 | "47","55408" 50 | "48","55401" 51 | "49","55417" 52 | "50","55405" 53 | "51","55417" 54 | "52","55405" 55 | "53","55104" 56 | "54","55402" 57 | "55","55408" 58 | "56","55409" 59 | "57","55410" 60 | "58","55419" 61 | "59","55408" 62 | "60","55418" 63 | "61","55420" 64 | "62","55130" 65 | "63","55418" 66 | "64","55404" 67 | "65","55107" 68 | "66","55101" 69 | "67","55420" 70 | "68","55408" 71 | "69","55130" 72 | "70","55411" 73 | "71","55403" 74 | "72","55413" 75 | "73","55108" 76 | "74","55101" 77 | "75","55411" 78 | "76","55420" 79 | "77","55101" 80 | "78","55104" 81 | "79","55407" 82 | "80","55103" 83 | "81","55420" 84 | "82","55418" 85 | "83","55420" 86 | "84","55408" 87 | "85","55104" 88 | "86","55415" 89 | "87","55417" 90 | "88","55411" 91 | "89","55413" 92 | "90","55417" 93 | "91","55414" 94 | "92","55419" 95 | "93","55106" 96 | "94","55130" 97 | "95","55417" 98 | "96","55417" 99 | "97","55407" 100 | "98","55102" 101 | "99","55407" 102 | -------------------------------------------------------------------------------- /demo/eos/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | # in case they were started 6 | (cd dashboards; docker compose down -v) 7 | (cd monitoring; docker compose down -v) 8 | 9 | (cd connect; docker compose down -v) 10 | (cd druid; docker compose down -v) 11 | (cd ksqlDB; docker compose down -v) 12 | (cd kafka-lb; docker compose down -v) 13 | (cd kafka; docker compose down -v) 14 | -------------------------------------------------------------------------------- /demo/eos/druid/order_store_zipcode.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_store_zipcode", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "STORE_ZIPCODE" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "HOUR", 50 | "queryGranularity": "minute", 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [] 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/eos/druid/order_uncommitted.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_uncommitted" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 4 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_uncommitted", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | { 34 | "type": "long", 35 | "name": "ORDER_ID" 36 | }, 37 | "USER_ID", 38 | "STORE_ID", 39 | "CITY", 40 | "STATE", 41 | "ZIPCODE" 42 | ] 43 | }, 44 | "metricsSpec": [ 45 | { 46 | "type": "count", 47 | "name": "COUNT" 48 | }, 49 | { 50 | "type": "longSum", 51 | "name": "QUANTITY", 52 | "fieldName": "QUANTITY" 53 | } 54 | ], 55 | "granularitySpec": { 56 | "type": "uniform", 57 | "segmentGranularity": "HOUR", 58 | "queryGranularity": null, 59 | "rollup": false, 60 | "intervals": null 61 | }, 62 | "transformSpec": { 63 | "transforms": [] 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /demo/eos/druid/order_user_reward.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 2 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_user_reward", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD" 34 | ] 35 | }, 36 | "metricsSpec": [ 37 | { 38 | "type": "count", 39 | "name": "COUNT" 40 | }, 41 | { 42 | "type": "longSum", 43 | "name": "QUANTITY", 44 | "fieldName": "QUANTITY" 45 | } 46 | ], 47 | "granularitySpec": { 48 | "type": "uniform", 49 | "segmentGranularity": "fifteen_minute", 50 | "queryGranularity": "minute", 51 | "rollup": true, 52 | "intervals": null 53 | }, 54 | "transformSpec": { 55 | "transforms": [] 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/eos/druid/order_user_reward_store_zipcode.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092,broker-2:9092,broker-3:9092", 8 | "isolation.level": "read_committed" 9 | }, 10 | "topic": "ORDERS_ENRICHED", 11 | "inputFormat": { 12 | "type": "json" 13 | }, 14 | "useEarliestOffset": true, 15 | "pollTimeout": 100, 16 | "period": "PT30S", 17 | "taskCount": 1 18 | }, 19 | "tuningConfig": { 20 | "type": "kafka", 21 | "maxRowsInMemory": 10000, 22 | "maxRowsPerSegment": 50000, 23 | "maxTotalRows": 200000 24 | }, 25 | "dataSchema": { 26 | "dataSource": "orders_user_reward_store_zipcode", 27 | "timestampSpec": { 28 | "column": "ORDER_TS", 29 | "format": "auto" 30 | }, 31 | "dimensionsSpec": { 32 | "dimensions": [ 33 | "USER_REWARD", 34 | "STORE_ZIPCODE" 35 | ] 36 | }, 37 | "metricsSpec": [ 38 | { 39 | "type": "count", 40 | "name": "COUNT" 41 | }, 42 | { 43 | "type": "longSum", 44 | "name": "QUANTITY", 45 | "fieldName": "QUANTITY" 46 | } 47 | ], 48 | "granularitySpec": { 49 | "type": "uniform", 50 | "segmentGranularity": "HOUR", 51 | "queryGranularity": "minute", 52 | "rollup": true, 53 | "intervals": null 54 | }, 55 | "transformSpec": { 56 | "transforms": [] 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /demo/eos/ksql/orders.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS ( 2 | order_id bigint, 3 | code varchar, 4 | user_id varchar, 5 | store_id varchar, 6 | quantity bigint, 7 | address struct 8 | ) with (kafka_topic='orders', value_format='json'); 9 | 10 | describe ORDERS extended; 11 | -------------------------------------------------------------------------------- /demo/eos/ksql/orders_enriched.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS_ENRICHED 2 | as 3 | select 4 | cast(O.ORDER_ID as string) as ORDER_KEY, 5 | O.ROWTIME as ORDER_TS, 6 | AS_VALUE(O.ORDER_ID) as ORDER_ID, 7 | O.CODE as CODE, 8 | O.USER_ID as USER_ID, 9 | U.REWARD as USER_REWARD, 10 | U.AREA_CODE as USER_AREA_CODE, 11 | O.STORE_ID as STORE_ID, 12 | S.ZIPCODE as STORE_ZIPCODE, 13 | O.QUANTITY as QUANTITY 14 | from ORDERS O 15 | join USERS U on U.USER_ID = O.USER_ID 16 | join STORES S on S.STORE_ID = O.STORE_ID 17 | partition by cast(O.ORDER_ID as string); 18 | 19 | describe ORDERS_ENRICHED extended; 20 | -------------------------------------------------------------------------------- /demo/eos/ksql/stores.ksql: -------------------------------------------------------------------------------- 1 | create or replace table STORES ( 2 | store_id varchar primary key, 3 | zipcode varchar 4 | ) with (kafka_topic='stores', value_format='json'); 5 | 6 | describe STORES extended; 7 | -------------------------------------------------------------------------------- /demo/eos/ksql/users.ksql: -------------------------------------------------------------------------------- 1 | create or replace table USERS ( 2 | user_id varchar primary key, 3 | reward varchar, 4 | area_code varchar 5 | ) with (kafka_topic='users', value_format='json'); 6 | 7 | describe USERS extended; 8 | -------------------------------------------------------------------------------- /demo/eos/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR=$(dirname $0) 4 | 5 | cd $DIR 6 | 7 | BIN=$DIR/../../bin 8 | 9 | CONNECT=$BIN/connect 10 | DRUID=$BIN/druid 11 | KSQL=$BIN/ksql-shell 12 | 13 | 14 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 15 | 16 | 17 | $CONNECT available 18 | [ $? -ne 0 ] && echo "connector RESTful API is not yet available, aborting script. wait until connector is ready to run this script." && exit 1 19 | 20 | 21 | # also done in build.sh 22 | cp -r data/* ../../connect/data 23 | 24 | 25 | kt --create --replication-factor 3 --partitions 4 --topic orders 26 | kt --create --replication-factor 3 --partitions 4 --topic ORDERS_ENRICHED 27 | kt --create --replication-factor 3 --partitions 4 --topic users 28 | kt --create --replication-factor 3 --partitions 4 --topic stores 29 | 30 | $CONNECT plugins 31 | [ $? -eq 1 ] && error_msg "connect RESTful API not yet available" && exit 1 32 | 33 | 34 | $CONNECT create ./connectors/store.json 35 | $CONNECT create ./connectors/user.json 36 | $CONNECT create ./connectors/order.json 37 | 38 | $KSQL ./ksql/users.ksql 39 | $KSQL ./ksql/stores.ksql 40 | $KSQL ./ksql/orders.ksql 41 | 42 | echo "sleeping 2 seconds to make sure the tables for users and stores are hydrated before orders enrichment stream is created" 43 | sleep 2 44 | $KSQL ./ksql/orders_enriched.ksql 45 | 46 | #$DRUID load ./druid/order.json 47 | -------------------------------------------------------------------------------- /demo/eos/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | # in case they were started 6 | (cd dashboards; docker compose stop) 7 | 8 | (cd connect; docker compose stop) 9 | (cd druid; docker compose stop) 10 | (cd ksqlDB; docker compose stop) 11 | (cd kafka-lb; docker compose stop) 12 | (cd kafka; docker compose stop) 13 | -------------------------------------------------------------------------------- /demo/eos/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | ./network.sh 6 | 7 | # if broker-4 is commented out, do not try to start it -- script technically will work if any broker is commented out 8 | ACTIVE_BROKERS=$(sed -n -E -e "s/^ (broker-[0-9]):/\1 /p" ./kafka/docker-compose.yml | tr -d '\n') 9 | (cd kafka; docker compose up -d zookeeper ${ACTIVE_BROKERS}) 10 | 11 | #(cd kafka-lb; docker compose up -d) 12 | 13 | (cd connect; docker compose up -d) 14 | 15 | (cd druid; docker compose up -d) 16 | 17 | (cd ksqlDB; docker compose up -d) 18 | 19 | #(cd monitoring; docker compose up -d prometheus grafana) 20 | -------------------------------------------------------------------------------- /demo/jdbc/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", 3 | "dialect.name" : "MySqlDatabaseDialect", 4 | "tasks.max": "1", 5 | "connection.url": "${file:/etc/kafka-connect/secrets/mysql.properties:CONNECTION_URL}", 6 | "connection.user" : "${file:/etc/kafka-connect/secrets/mysql.properties:CONNECTION_USER}", 7 | "connection.password" : "${file:/etc/kafka-connect/secrets/mysql.properties:CONNECTION_PASSWORD}", 8 | 9 | "tables" : "SEQ_EXAMPLE", 10 | "topic.prefix": "MAIN", 11 | "schema.pattern": "MAIN", 12 | 13 | "poll.interval.ms" : 6000, 14 | "mode":"incrementing", 15 | "_mode":"bulk", 16 | "_pk.mode": "key+value", 17 | 18 | "validate.non.null": true, 19 | "_key.converter.schemas.enable": false, 20 | 21 | 22 | "_transforms" : "SetKeySchemaName,SetValueSchemaName", 23 | "transforms.SetKeySchemaName.type": "org.apache.kafka.connect.transforms.SetSchemaMetadata$Key", 24 | "transforms.SetKeySchemaName.schema.name": "FOO", 25 | "transforms.SetValueSchemaName.type": "org.apache.kafka.connect.transforms.SetSchemaMetadata$Value", 26 | "transforms.SetValueSchemaName.schema.name": "BAR" 27 | 28 | } 29 | 30 | -------------------------------------------------------------------------------- /demo/metrics/druid/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092" 8 | }, 9 | "topic": "_metrics-kafka-streams", 10 | "inputFormat": { 11 | "type": "json" 12 | }, 13 | "useEarliestOffset": true 14 | }, 15 | "tuningConfig": { 16 | "type": "kafka" 17 | }, 18 | "dataSchema": { 19 | "dataSource": "_metrics-kafka-streams", 20 | "granularitySpec": { 21 | "type": "uniform", 22 | "queryGranularity": "fifteen_minute", 23 | "segmentGranularity": "fifteen_minute", 24 | "rollup": true 25 | }, 26 | "timestampSpec": { 27 | "column": "timestamp", 28 | "format": "millis" 29 | }, 30 | "dimensionsSpec": { 31 | "dimensions": [ 32 | "group", 33 | "group-id", 34 | "client-id", 35 | "thread-id", 36 | "task-id", 37 | "processor-node-id", 38 | "partition", 39 | "subtopology", 40 | "topic", 41 | "record-cache-id", 42 | "rocksdb-state", 43 | "name" 44 | ] 45 | }, 46 | "metricsSpec": [ 47 | { 48 | "name": "count", 49 | "type": "count" 50 | }, 51 | { 52 | "name": "value", 53 | "type": "doubleLast", 54 | "fieldName": "value" 55 | } 56 | ] 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/mongo-cdc/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | 5 | touch ../tmp/connect-distributed.properties 6 | 7 | 8 | if [ ! -d "../../connect/jars/debezium-debezium-connector-mongodb" ]; then 9 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt debezium/debezium-connector-mongodb:1.9.2 10 | fi 11 | 12 | if [ ! -d "../../connect/jars/mongodb-kafka-connect-mongodb" ]; then 13 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt mongodb/kafka-connect-mongodb:1.7.0 14 | fi 15 | 16 | 17 | rm -f ../tmp/connect-distributed.properties 18 | 19 | -------------------------------------------------------------------------------- /demo/mongo-cdc/connectors/mongo-cdc.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class" : "io.debezium.connector.mongodb.MongoDbConnector", 3 | 4 | "tasks.max": "1", 5 | 6 | "mongodb.hosts": "rs0/mongo:27017", 7 | "mongodb.user": "root", 8 | "mongodb.password": "mongo", 9 | 10 | "mongodb.name": "restheart", 11 | 12 | "database.history.kafka.bootstrap.servers" : "broker-1:9092", 13 | 14 | "_key.converter": "org.apache.kafka.connect.storage.StringConverter", 15 | "key.converter": "org.apache.kafka.connect.json.JsonConverter", 16 | "key.converter.schemas.enable": false, 17 | 18 | "_value.converter": "org.apache.kafka.connect.storage.StringConverter", 19 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 20 | "value.converter.schemas.enable": false, 21 | 22 | "transforms" : "flatten,unwrap", 23 | 24 | "transforms.flatten.type" : "org.apache.kafka.connect.transforms.Flatten$Key", 25 | "transforms.flatten.blacklist" : "id", 26 | 27 | "transforms.key_unwrap.type": "io.debezium.connector.mongodb.transforms.ExtractNewDocumentState", 28 | 29 | "transforms.unwrap.type": "io.debezium.connector.mongodb.transforms.ExtractNewDocumentState", 30 | 31 | "transforms.after.type" : "org.apache.kafka.connect.transforms.Flatten$Value", 32 | "transforms.after.blacklist" : "after" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /demo/mongo-cdc/connectors/mongo.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "com.mongodb.kafka.connect.MongoSinkConnector", 3 | "tasks.max": 1, 4 | 5 | "connection.uri": "mongodb://root:mongo@mongo:27017", 6 | "database": "restheart", 7 | "collection": "DEF", 8 | 9 | "topics": "restheart.restheart.ABC", 10 | 11 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 12 | "key.converter.schemas.enable": "false", 13 | 14 | "_value.converter": "org.apache.kafka.connect.storage.StringConverter", 15 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 16 | "value.converter.schemas.enable": "false", 17 | 18 | "errors.tolerance": "all", 19 | "errors.deadletterqueue.topic.name":"connect.sink.dlt.S3SinkConnector", 20 | "errors.log.include.messages" : true, 21 | "errors.deadletterqueue.context.headers.enable" : true, 22 | "errors.deadletterqueue.topic.replication.factor": 3 23 | 24 | } 25 | -------------------------------------------------------------------------------- /demo/mongo-cdc/data/orders.csv: -------------------------------------------------------------------------------- 1 | order_id,store_id,user_id,quantity 2 | 0,0,0,200 3 | 1,1,2,300 4 | 2,2,2,100 5 | 3,0,1,400 6 | -------------------------------------------------------------------------------- /demo/mongo-cdc/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | (cd ksqlDB; docker compose down -v) 6 | (cd connect; docker compose down -v) 7 | (cd kafka; docker compose down -v) 8 | 9 | (cd mongo; docker compose down -v) 10 | -------------------------------------------------------------------------------- /demo/mongo-cdc/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 1 ]; then 4 | COUNT=1 5 | else 6 | COUNT=$1 7 | shift 8 | fi 9 | 10 | function number() { 11 | echo $(($RANDOM % $1)) 12 | } 13 | 14 | function random() { 15 | echo $(openssl rand -hex 5) 16 | } 17 | 18 | for i in $(seq $COUNT); do 19 | 20 | order_id=$(random) 21 | store_id=$(number 100) 22 | user_id=$(number 1000) 23 | quantity=$(number 9999) 24 | ts=$(date +"%Y-%m-%d %H:%M:%S") 25 | 26 | echo "creating order order_id=${order_id}, store_id=${store_id}, user_id=${user_id}, quantity=${quantity}, ts=${ts}" 27 | 28 | DOC="{ \"order_id\" : \"${order_id}\" }" 29 | 30 | #curl -i -H "Content-Type:application/json" -X POST -u "admin:restheart" http://localhost:28080/ABC -d "${DOC}" 31 | curl -i -H "Content-Type:application/json" -X POST -u "admin:secret" http://localhost:28080/ABC -d "${DOC}" 32 | 33 | # sleep 0.1 34 | 35 | done 36 | -------------------------------------------------------------------------------- /demo/mongo-cdc/ksql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR=$(dirname $0) 4 | cd $DIR 5 | 6 | BIN=$DIR/../../bin 7 | KSQL=$BIN/ksql-shell 8 | 9 | $KSQL ./ksql/users.ksql 10 | $KSQL ./ksql/stores.ksql 11 | $KSQL ./ksql/orders.ksql 12 | -------------------------------------------------------------------------------- /demo/mongo-cdc/ksql/orders.ksql: -------------------------------------------------------------------------------- 1 | create or replace stream ORDERS ( 2 | order_id varchar, 3 | user_id varchar, 4 | store_id varchar, 5 | quantity bigint 6 | ) with (kafka_topic='mysql_main.MAIN.orders', value_format='avro', key_format='kafka'); 7 | 8 | describe ORDERS extended; 9 | -------------------------------------------------------------------------------- /demo/mongo-cdc/ksql/orders_enriched.ksql: -------------------------------------------------------------------------------- 1 | create stream ORDERS_ENRICHED with(VALUE_FORMAT='json') 2 | as 3 | select 4 | cast(O.ORDER_ID as string) as ORDER_KEY, 5 | O.ROWTIME as ORDER_TS, 6 | AS_VALUE(O.ORDER_ID) as ORDER_ID, 7 | O.USER_ID as USER_ID, 8 | U.USER_NAME as USER_NAME, 9 | O.STORE_ID as STORE_ID, 10 | S.STORE_NAME as STORE_NAME, 11 | O.QUANTITY as QUANTITY 12 | from ORDERS O 13 | join USERS U on U.USER_ID = O.USER_ID 14 | join STORES S on S.STORE_ID = O.STORE_ID 15 | partition by cast(O.ORDER_ID as string); 16 | 17 | describe ORDERS_ENRICHED extended; 18 | -------------------------------------------------------------------------------- /demo/mongo-cdc/ksql/stores.ksql: -------------------------------------------------------------------------------- 1 | create or replace table STORES ( 2 | store_id varchar primary key, 3 | store_name varchar 4 | ) with (kafka_topic='mysql_main.MAIN.stores', value_format='avro', key_format='kafka'); 5 | 6 | describe STORES extended; 7 | -------------------------------------------------------------------------------- /demo/mongo-cdc/ksql/users.ksql: -------------------------------------------------------------------------------- 1 | create or replace table USERS ( 2 | user_id varchar primary key, 3 | user_name varchar 4 | ) with (kafka_topic='mysql_main.MAIN.users', value_format='avro', key_format='kafka'); 5 | 6 | describe USERS extended; 7 | -------------------------------------------------------------------------------- /demo/mongo-cdc/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR=$(dirname $0) 4 | cd $DIR 5 | BIN=$DIR/../../bin 6 | CONNECT=$BIN/connect 7 | 8 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 9 | 10 | $CONNECT available 11 | [ $? -ne 0 ] && echo "connector RESTful API is not yet available, aborting script. wait until connector is ready to run this script." && exit 1 12 | 13 | #kt --create --replication-factor 3 --partitions 4 --topic mysql_main.MAIN.users 14 | #kt --create --replication-factor 3 --partitions 4 --topic mysql_main.MAIN.stores 15 | #kt --create --replication-factor 3 --partitions 4 --topic mysql_main.MAIN.orders 16 | 17 | #restheart.restheart.ABC 18 | #restheart.restheart.ABC2 19 | #restheart.restheart.DEF 20 | 21 | 22 | $CONNECT create ./connectors/mongo-cdc.json 23 | $CONNECT status ./connectors/mongo-cdc.json 24 | 25 | $CONNECT create ./connectors/mongo.json 26 | $CONNECT status ./connectors/mongo.json 27 | -------------------------------------------------------------------------------- /demo/mongo-cdc/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | ./network.sh 6 | 7 | (cd kafka; docker compose up -d $(docker-compose config --services | grep -v schema-registry)) 8 | (cd connect; docker compose up -d) 9 | (cd mongo; docker compose up -d) 10 | 11 | #(cd ksqlDB; docker compose up -d) 12 | 13 | # takes the longest, but want to let others start while it is starting, so don't wait until evertyhing else has been started. 14 | (cd connect; docker compose up -d --wait) 15 | 16 | -------------------------------------------------------------------------------- /demo/mongo/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | 5 | touch ../tmp/connect-distributed.properties 6 | 7 | 8 | if [ ! -d "../../connect/jars/debezium-debezium-connector-mongodb" ]; then 9 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt debezium/debezium-connector-mongodb:1.8.1 10 | fi 11 | 12 | if [ ! -d "../../connect/jars/mongodb-kafka-connect-mongodb" ]; then 13 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt mongodb/kafka-connect-mongodb:1.7.0 14 | fi 15 | 16 | if [ ! -d "../../connect/jars/debezium-debezium-connector-mysql" ]; then 17 | confluent-hub install --worker-configs ../tmp/connect-distributed.properties --component-dir ../../connect/jars --no-prompt debezium/debezium-connector-mysql:1.7.1 18 | fi 19 | 20 | 21 | rm -f ../tmp/connect-distributed.properties 22 | 23 | -------------------------------------------------------------------------------- /demo/mongo/connectors/mongo.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "com.mongodb.kafka.connect.MongoSinkConnector", 3 | "tasks.max": 1, 4 | 5 | "connection.uri": "mongodb://mongo:27017", 6 | "database": "_DATABASE_", 7 | "collection": "_COLLECTION_", 8 | 9 | "topics": "ORDERS_ENRICHED", 10 | 11 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 12 | "key.converter.schemas.enable": "false", 13 | 14 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 15 | "value.converter.schemas.enable": "false" 16 | 17 | } 18 | -------------------------------------------------------------------------------- /demo/mongo/data/orders.csv: -------------------------------------------------------------------------------- 1 | order_id,store_id,user_id,quantity 2 | 0,0,0,200 3 | 1,1,2,300 4 | 2,2,2,100 5 | 3,0,1,400 6 | -------------------------------------------------------------------------------- /demo/mongo/db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR=$(dirname $0) 4 | cd $DIR 5 | 6 | cp -r data/* ../../mysql/data 7 | 8 | cat > ../../mysql/data/load.sql < ../../mysql/data/schema.sql < ../../../connect/secrets/opensky.properties 26 | USERNAME= 27 | PASSWORD= 28 | EOF 29 | 30 | fi 31 | 32 | # 33 | # Ksql User Defined Functions - Geo 34 | # 35 | 36 | if [ ! -d ksqldb-udf-geospatial ]; then 37 | git clone https://github.com/nbuesing/ksqldb-udf-geospatial.git 38 | fi 39 | 40 | (cd ksqldb-udf-geospatial; git pull; ./gradlew --no-daemon build --warning-mode all) 41 | 42 | cp ./ksqldb-udf-geospatial/build/libs/ksqldb-udf-geospatial-1.0.0-all.jar ../../../ksqldb/ext 43 | -------------------------------------------------------------------------------- /demo/opensky/connectors/opensky.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "com.github.nbuesing.kafka.connect.opensky.OpenSkySourceConnector", 3 | 4 | "tasks.max": "1", 5 | 6 | "topic": "flightdata", 7 | 8 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 9 | 10 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 11 | "value.converter.schemas.enable": "false", 12 | 13 | "interval": "60", 14 | 15 | "_USA_bounding.boxes": "45.8389 47.8229 5.9962 10.5226 , 24.396308 49.384358 -124.848974 -66.885444", 16 | "_NA_AND_INDIA_bounding.boxes": "10.008944 74.924300 -171.386719 -47.812500, 3.062117 35.664915 62.964567 94.077848", 17 | 18 | "opensky.timeout.connect": "20s", 19 | "opensky.timeout.read": "40s", 20 | 21 | "_opensky": "enable username and password if you have credentials", 22 | "_opensky.username": "${file:/etc/kafka-connect/secrets/opensky.properties:USERNAME}", 23 | "_opensky.password": "${file:/etc/kafka-connect/secrets/opensky.properties:PASSWORD}", 24 | 25 | 26 | "transforms": "flatten,rename", 27 | "transforms.flatten.type": "org.apache.kafka.connect.transforms.Flatten$Value", 28 | "transforms.flatten.delimiter": "_", 29 | "transforms.rename.type": "org.apache.kafka.connect.transforms.ReplaceField$Value", 30 | "transforms.rename.renames": "location_lat:latitude,location_lon:longitude" 31 | } 32 | -------------------------------------------------------------------------------- /demo/opensky/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | (cd connect; docker compose stop) 6 | (cd dashboards; docker compose stop) 7 | (cd druid; docker compose stop) 8 | (cd ksqlDB; docker compose stop) 9 | (cd kafka; docker compose stop) 10 | -------------------------------------------------------------------------------- /demo/opensky/druid/flight_summary.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092" 8 | }, 9 | "topic": "FLIGHTS_SUMMARY", 10 | "inputFormat": { 11 | "type": "json" 12 | }, 13 | "useEarliestOffset": true, 14 | "taskCount": 4 15 | }, 16 | "tuningConfig": { 17 | "type": "kafka" 18 | }, 19 | "dataSchema": { 20 | "dataSource": "flight_summary", 21 | "timestampSpec": { 22 | "column": "TIMEPOSITION", 23 | "format": "auto" 24 | }, 25 | "dimensionsSpec": { 26 | "dimensions": [ 27 | "CALLSIGN", 28 | "PATH", 29 | "NEAREST_AIRPORT", 30 | "NEAREST_REGION", 31 | "NEAREST_CITY", 32 | { 33 | "type": "long", 34 | "name": "LASTCONTACT" 35 | }, 36 | { 37 | "type": "double", 38 | "name": "LATITUDE" 39 | }, 40 | { 41 | "type": "double", 42 | "name": "LONGITUDE" 43 | }, 44 | { 45 | "type": "double", 46 | "name": "ALTITUDE" 47 | }, 48 | { 49 | "type": "double", 50 | "name": "VELOCITY" 51 | }, 52 | { 53 | "type": "double", 54 | "name": "DISTANCE" 55 | } 56 | ] 57 | }, 58 | "metricsSpec": [ 59 | { 60 | "type": "count", 61 | "name": "count" 62 | } 63 | ], 64 | "granularitySpec": { 65 | "type": "uniform", 66 | "segmentGranularity": "HOUR", 67 | "queryGranularity": null, 68 | "rollup": false, 69 | "intervals": null 70 | }, 71 | "transformSpec": { 72 | "transforms": [] 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /demo/opensky/druid/flight_summary_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "kafka", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "kafka", 6 | "consumerProperties": { 7 | "bootstrap.servers": "broker-1:9092" 8 | }, 9 | "topic": "FLIGHTS_SUMMARY", 10 | "inputFormat": { 11 | "type": "json" 12 | }, 13 | "useEarliestOffset": true, 14 | "taskCount": 4 15 | }, 16 | "tuningConfig": { 17 | "type": "kafka" 18 | }, 19 | "dataSchema": { 20 | "dataSource": "flight_summary", 21 | "timestampSpec": { 22 | "column": "TIMEPOSITION", 23 | "format": "auto" 24 | }, 25 | "dimensionsSpec": { 26 | "dimensions": [ 27 | "CALLSIGN", 28 | "PATH", 29 | "NEAREST_AIRPORT", 30 | "NEAREST_REGION", 31 | "NEAREST_CITY", 32 | { 33 | "type": "long", 34 | "name": "LASTCONTACT" 35 | }, 36 | { 37 | "type": "double", 38 | "name": "LATITUDE" 39 | }, 40 | { 41 | "type": "double", 42 | "name": "LONGITUDE" 43 | }, 44 | { 45 | "type": "double", 46 | "name": "ALTITUDE" 47 | }, 48 | { 49 | "type": "double", 50 | "name": "VELOCITY" 51 | }, 52 | { 53 | "type": "double", 54 | "name": "DISTANCE" 55 | } 56 | ] 57 | }, 58 | "metricsSpec": [ 59 | { 60 | "type": "count", 61 | "name": "count" 62 | } 63 | ], 64 | "granularitySpec": { 65 | "type": "uniform", 66 | "segmentGranularity": "HOUR", 67 | "queryGranularity": null, 68 | "rollup": false, 69 | "intervals": null 70 | }, 71 | "transformSpec": { 72 | "transforms": [] 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /demo/opensky/ksql/config.properties: -------------------------------------------------------------------------------- 1 | auto.offset.reset=earliest 2 | -------------------------------------------------------------------------------- /demo/opensky/ksql/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | FILE=$1 4 | shift 5 | 6 | ksql --config-file config.properties --file=$FILE 7 | 8 | 9 | -------------------------------------------------------------------------------- /demo/opensky/ksql/flights.ksql: -------------------------------------------------------------------------------- 1 | create stream FLIGHTS ( 2 | id varchar KEY, 3 | callsign varchar, 4 | originCountry varchar, 5 | timePosition bigint, 6 | lastContact bigint, 7 | latitude double, 8 | longitude double, 9 | barometricAltitude double, 10 | geometricAltitude double, 11 | onGround boolean, 12 | heading double, 13 | velocity double 14 | ) with (kafka_topic='flightdata', value_format='json', timestamp='lastContact'); 15 | 16 | describe FLIGHTS extended; 17 | -------------------------------------------------------------------------------- /demo/opensky/ksql/flights_summary.ksql: -------------------------------------------------------------------------------- 1 | create table FLIGHTS_SUMMARY as 2 | select 3 | id, 4 | latest_by_offset(callsign) callsign, 5 | max(timeposition) timeposition, 6 | max(lastcontact) lastcontact, 7 | latest_by_offset(velocity) velocity, 8 | latest_by_offset(velocity) heading, 9 | latest_by_offset(latitude) latitude, 10 | latest_by_offset(longitude) longitude, 11 | latest_by_offset(geometricAltitude) altitude, 12 | geopoint_segment_distance(geopoint(latitude, longitude, geometricaltitude, timeposition)) distance, 13 | latest_by_offset(nearest_airport(latitude, longitude)->code) nearest_aiport, 14 | latest_by_offset(nearest_city(latitude, longitude)->name) nearest_city, 15 | latest_by_offset(nearest_city(latitude, longitude)->iso) nearest_region 16 | from flights 17 | window session(30 minutes) 18 | where onGround = false 19 | group by id; 20 | 21 | describe FLIGHTS_SUMMARY extended; 22 | -------------------------------------------------------------------------------- /demo/opensky/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR=$(dirname $0) 4 | 5 | cd $DIR 6 | 7 | BIN=$DIR/../../bin 8 | 9 | CONNECT=$BIN/connect 10 | DRUID=$BIN/druid 11 | KSQL=$BIN/ksql-shell 12 | 13 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 14 | 15 | # 16 | # connect startup is usually the last service up from those that are not dependent on others, so we make sure that is up before we run through the script. 17 | # 18 | # if/when other services end up being a startup-issue, checks for them should be considered. 19 | # 20 | 21 | $CONNECT available 22 | [ $? -ne 0 ] && echo "connector RESTful API is not yet available, aborting script. wait until connector is ready to run this script." && exit 1 23 | 24 | 25 | kt --create --replication-factor 3 --partitions 4 --topic flightdata 26 | kt --create --replication-factor 3 --partitions 4 --topic FLIGHTS_SUMMARY 27 | 28 | $CONNECT create ./connectors/opensky.json 29 | 30 | $KSQL ./ksql/flights.ksql 31 | sleep 1 32 | $KSQL ./ksql/flights_summary.ksql 33 | 34 | 35 | $DRUID load ./druid/flight_summary.json 36 | 37 | ./superset/create.sh 38 | -------------------------------------------------------------------------------- /demo/opensky/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | (cd connect; docker compose down -v) 6 | (cd dashboards; docker compose down -v) 7 | (cd druid; docker compose down -v) 8 | (cd ksqlDB; docker compose down -v) 9 | (cd kafka; docker compose down -v) 10 | -------------------------------------------------------------------------------- /demo/opensky/superset/chart_country.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "cache_timeout": 0, 4 | "dashboards" : [], 5 | "datasource_name" : "flight_summary", 6 | "datasource_id" : #DATASOURCE_ID#, 7 | "datasource_type" : "table", 8 | "description" : "#COUNTRY#", 9 | "slice_name": "flight_summary_#COUNTRY#", 10 | "viz_type": "country_map", 11 | 12 | "params": "{\n \"adhoc_filters\": [],\n \"datasource\": \"#DATASOURCE_ID#__table\",\n \"entity\": \"NEAREST_REGION\",\n \"extra_form_data\": {},\n \"granularity_sqla\": \"__time\",\n \"linear_color_scheme\": \"superset_seq_1\",\n \"metric\": \"count\",\n \"number_format\": \"SMART_NUMBER\",\n \"select_country\": \"#COUNTRY#\",\n \"time_range\": \"DATEADD(DATETRUNC(DATETIME(\\\"now\\\"), MINUTE), -2, minute) : DATEADD(DATETRUNC(DATETIME(\\\"now\\\"), MINUTE), -1, minute) \",\n \"time_range_endpoints\": [\n \"inclusive\",\n \"exclusive\"\n ],\n \"url_params\": {},\n \"viz_type\": \"country_map\"\n}", 13 | 14 | "query_context": "{\"datasource\":{\"id\":1,\"type\":\"table\"},\"force\":false,\"queries\":[{\"time_range\":\"DATEADD(DATETRUNC(DATETIME(\\\"now\\\"), MINUTE), -2, minute) : DATEADD(DATETRUNC(DATETIME(\\\"now\\\"), MINUTE), -1, minute)\",\"granularity\":\"__time\",\"filters\":[],\"extras\":{\"time_range_endpoints\":[\"inclusive\",\"exclusive\"],\"having\":\"\",\"having_druid\":[],\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[\"count\"],\"annotation_layers\":[],\"timeseries_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{}}],\"result_format\":\"json\",\"result_type\":\"full\"}" 15 | 16 | 17 | } 18 | -------------------------------------------------------------------------------- /demo/opensky/superset/druid.json: -------------------------------------------------------------------------------- 1 | { 2 | "allow_csv_upload": true, 3 | "allow_ctas": true, 4 | "allow_cvas": true, 5 | "allow_dml": true, 6 | "allow_multi_schema_metadata_fetch": true, 7 | "allow_run_async": true, 8 | "cache_timeout": 0, 9 | "configuration_method": "sqlalchemy_form", 10 | "database_name": "druid", 11 | "expose_in_sqllab": true, 12 | "impersonate_user": true, 13 | "parameters": {}, 14 | "sqlalchemy_uri": "druid://druid-router:8888/druid/v2/sql" 15 | } 16 | -------------------------------------------------------------------------------- /demo/opensky/superset/login.json: -------------------------------------------------------------------------------- 1 | { 2 | "password": "admin", 3 | "provider": "db", 4 | "refresh": true, 5 | "username": "admin" 6 | } 7 | -------------------------------------------------------------------------------- /demo/opensky/tmp/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/demo/opensky/tmp/.gitkeep -------------------------------------------------------------------------------- /demo/opensky/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | if [[ ! -d ./connect/jars/kafka-connect-opensky || ! -f ./ksqldb/ext/ksqldb-udf-geospatial-1.0.0-all.jar ]]; then 6 | echo "" 7 | echo "" 8 | echo " run build.sh to build opensky connector and geospatial functions for KsqlDB and deploy them." 9 | echo "" 10 | echo "" 11 | exit 12 | fi 13 | 14 | ./network.sh 15 | 16 | (cd kafka; docker compose up -d $(docker-compose config --services | grep -v schema-registry)) 17 | 18 | (cd connect; docker compose up -d) 19 | 20 | (cd ksqlDB; docker compose up -d) 21 | 22 | (cd druid; docker compose up -d) 23 | 24 | (cd dashboards; docker compose up -d) 25 | 26 | -------------------------------------------------------------------------------- /demo/rollups/druid/store_rewards_user.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "index_parallel", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "index_parallel", 6 | "inputSource": { 7 | "type": "local", 8 | "baseDir": "/opt/staging", 9 | "filter": "*.csv" 10 | }, 11 | "inputFormat": { 12 | "type": "csv", 13 | "findColumnsFromHeader": true 14 | } 15 | }, 16 | "tuningConfig": { 17 | "type": "index_parallel", 18 | "partitionsSpec": { 19 | "type": "hashed" 20 | }, 21 | "forceGuaranteedRollup": true 22 | }, 23 | "dataSchema": { 24 | "dataSource": "rollup_store_rewards_user", 25 | "timestampSpec": { 26 | "column": "time", 27 | "format": "iso" 28 | }, 29 | "granularitySpec": { 30 | "queryGranularity": "hour", 31 | "rollup": true, 32 | "segmentGranularity": "hour" 33 | }, 34 | "metricsSpec": [ 35 | { 36 | "name": "count", 37 | "type": "count" 38 | }, 39 | { 40 | "name": "quantity", 41 | "type": "longSum", 42 | "fieldName": "quantity" 43 | }, 44 | { 45 | "name": "amount", 46 | "type": "doubleSum", 47 | "fieldName": "amount" 48 | } 49 | ], 50 | "dimensionsSpec": { 51 | "dimensions": [ 52 | "store_name", 53 | "rewards", 54 | "user" 55 | ] 56 | } 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /demo/rollups/druid/store_rewards_user_unique.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "index_parallel", 3 | "spec": { 4 | "ioConfig": { 5 | "type": "index_parallel", 6 | "inputSource": { 7 | "type": "local", 8 | "baseDir": "/opt/staging", 9 | "filter": "*.csv" 10 | }, 11 | "inputFormat": { 12 | "type": "csv", 13 | "findColumnsFromHeader": true 14 | } 15 | }, 16 | "tuningConfig": { 17 | "type": "index_parallel", 18 | "partitionsSpec": { 19 | "type": "hashed" 20 | }, 21 | "forceGuaranteedRollup": true 22 | }, 23 | "dataSchema": { 24 | "dataSource": "rollup_store_rewards_user_unique", 25 | "timestampSpec": { 26 | "column": "time", 27 | "format": "iso" 28 | }, 29 | "granularitySpec": { 30 | "queryGranularity": "minute", 31 | "rollup": true, 32 | "segmentGranularity": "hour" 33 | }, 34 | "metricsSpec": [ 35 | { 36 | "name": "count", 37 | "type": "count" 38 | }, 39 | { 40 | "name": "quantity", 41 | "type": "longSum", 42 | "fieldName": "quantity" 43 | }, 44 | { 45 | "name": "amount", 46 | "type": "doubleSum", 47 | "fieldName": "amount" 48 | }, 49 | { 50 | "name": "user_sketch", 51 | "type": "thetaSketch", 52 | "fieldName": "user" 53 | } 54 | ], 55 | "dimensionsSpec": { 56 | "dimensions": [ 57 | "store_name", 58 | "rewards" 59 | ] 60 | } 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /demo/storage/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | (cd storage; docker compose down -v) 6 | (cd connect; docker compose down -v) 7 | (cd kafka; docker compose down -v) 8 | -------------------------------------------------------------------------------- /demo/storage/f.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "FileStreamSink", 3 | "tasks.max": 1, 4 | "file": "test-distributed.txt", 5 | "topics": "s3_topic" 6 | } 7 | -------------------------------------------------------------------------------- /demo/storage/gcp.json: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class": "io.aiven.kafka.connect.gcs.GcsSinkConnector", 3 | 4 | "tasks.max": "1", 5 | 6 | "topics": "s3_topic", 7 | 8 | "gcs.bucket.name" : "ws30-in.rilldata.io", 9 | 10 | "gcs.credentials.path" : "/etc/kafka-connect/jars/key.json", 11 | 12 | "format.output.type" : "jsonl", 13 | "format.output.envelope": false, 14 | 15 | "format.output.fields": "value", 16 | "_format.output.fields": "key,value,offset,timestamp,headers", 17 | 18 | "file.name.prefix" : "temp/", 19 | "file.compression.type" : "gzip", 20 | "file.name.timestamp.timezone" : "UTC", 21 | "file.name.timestamp.source" : "wallclock", 22 | "file.name.template" : "y={{timestamp:unit=yyyy}}/m={{timestamp:unit=MM}}/d={{timestamp:unit=dd}}/H={{timestamp:unit=HH}}/{{topic}}-{{partition:padding=true}}-{{start_offset:padding=true}}.gz", 23 | 24 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 25 | "_key.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", 26 | "key.converter.schemas.enable": false, 27 | 28 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 29 | "_value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", 30 | "__value.converter": "org.apache.kafka.connect.storage.StringConverter", 31 | "value.converter.schemas.enable": false, 32 | 33 | "errors.tolerance": "all", 34 | "errors.deadletterqueue.topic.name":"connect.sink.dlt.GcsSinkConnector", 35 | "errors.log.include.messages" : true, 36 | "errors.deadletterqueue.context.headers.enable" : true, 37 | "errors.deadletterqueue.topic.replication.factor": 3 38 | 39 | } 40 | -------------------------------------------------------------------------------- /demo/storage/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/../.. 4 | 5 | ./network.sh 6 | 7 | (cd kafka; docker compose up -d zookeeper broker-1 broker-2 broker-3 broker-4) 8 | 9 | (cd connect; docker compose up -d) 10 | 11 | (cd storage; docker compose up -d) 12 | 13 | -------------------------------------------------------------------------------- /demo/tmp/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/demo/tmp/.gitkeep -------------------------------------------------------------------------------- /druid/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=druid 2 | -------------------------------------------------------------------------------- /druid/jmx_prometheus/druid-config.yml: -------------------------------------------------------------------------------- 1 | 2 | lowercaseOutputName: true 3 | rules: 4 | - pattern: java.lang 5 | -------------------------------------------------------------------------------- /druid/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/druid/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /druid/staging/exports/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/druid/staging/exports/.gitkeep -------------------------------------------------------------------------------- /elasticsearch/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=elasticsearch 2 | -------------------------------------------------------------------------------- /elasticsearch/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | services: 10 | 11 | elasticsearch: 12 | image: docker.elastic.co/elasticsearch/elasticsearch:8.2.1 13 | hostname: elasticsearch 14 | container_name: elasticsearch 15 | ports: 16 | - 9200:9200 17 | - 9300:9300 18 | healthcheck: 19 | test: curl -s -f http://localhost:9200/_cat/health || exit 1 20 | interval: 15s 21 | timeout: 5s 22 | retries: 20 23 | start_period: 15s 24 | environment: 25 | xpack.security.enabled: "false" 26 | discovery.type: single-node 27 | ES_JAVA_OPTS: "-Xms512m -Xmx512m" 28 | 29 | kibana: 30 | image: docker.elastic.co/kibana/kibana:8.2.1 31 | hostname: kibana 32 | container_name: kibana 33 | depends_on: 34 | - elasticsearch 35 | ports: 36 | - 5601:5601 37 | healthcheck: 38 | test: curl -s -f http://localhost:5601/login || exit 1 39 | interval: 15s 40 | timeout: 5s 41 | retries: 20 42 | start_period: 15s 43 | environment: 44 | xpack.security.enabled: "false" 45 | discovery.type: "single-node" 46 | -------------------------------------------------------------------------------- /flink/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=flink 2 | -------------------------------------------------------------------------------- /flink/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | services: 10 | jobmanager: 11 | image: flink:1.15.0-scala_2.12-java11 12 | container_name: flink_jobmanager 13 | ports: 14 | - "48081:8081" 15 | - "46123:6123" 16 | - "46124:6124" 17 | - "46125:6125" 18 | healthcheck: 19 | test: curl --fail http://localhost:8081/overview 20 | interval: 15s 21 | timeout: 5s 22 | retries: 30 23 | start_period: 0s 24 | command: jobmanager 25 | environment: 26 | - | 27 | FLINK_PROPERTIES= 28 | jobmanager.rpc.address: jobmanager 29 | volumes: 30 | - ./jars:/jars 31 | - ./data:/data 32 | - ./tmp:/Users/buesing/workspaces/gorillio/beam-pipelines/pipeline 33 | 34 | taskmanager: 35 | image: flink:1.15.0-scala_2.12-java11 36 | depends_on: 37 | - jobmanager 38 | command: taskmanager 39 | scale: 2 40 | healthcheck: 41 | test: exit 0 42 | interval: 15s 43 | timeout: 5s 44 | retries: 30 45 | start_period: 5s 46 | environment: 47 | - | 48 | FLINK_PROPERTIES= 49 | jobmanager.rpc.address: jobmanager 50 | taskmanager.numberOfTaskSlots: 2 51 | parallelism.default: 2 52 | taskmanager.memory.network.max: 268435460b 53 | taskmanager.memory.network.min: 268435460b 54 | taskmanager.memory.network.fraction: .4 55 | cluster.evenly-spread-out-slots: true 56 | volumes: 57 | - ./jars:/jars 58 | - ./data:/data 59 | # - ./tmp:/Users/buesing/workspaces/gorillio/beam-pipelines/pipeline 60 | -------------------------------------------------------------------------------- /kafka-lb/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=kafka_lb 2 | -------------------------------------------------------------------------------- /kafka-lb/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | services: 10 | 11 | broker-1-lb: 12 | # image: nginx:latest 13 | build: ./nginx 14 | container_name: broker-1-lb 15 | hostname: broker-1-lb 16 | cap_add: 17 | - NET_ADMIN 18 | volumes: 19 | - ./nginx/broker-1.conf:/etc/nginx/nginx.conf 20 | 21 | broker-2-lb: 22 | #image: nginx:latest 23 | build: ./nginx 24 | container_name: broker-2-lb 25 | hostname: broker-2-lb 26 | cap_add: 27 | - NET_ADMIN 28 | volumes: 29 | - ./nginx/broker-2.conf:/etc/nginx/nginx.conf 30 | 31 | broker-3-lb: 32 | #image: nginx:latest 33 | build: ./nginx 34 | container_name: broker-3-lb 35 | hostname: broker-3-lb 36 | cap_add: 37 | - NET_ADMIN 38 | volumes: 39 | - ./nginx/broker-3.conf:/etc/nginx/nginx.conf 40 | 41 | broker-4-lb: 42 | #image: nginx:latest 43 | build: ./nginx 44 | container_name: broker-4-lb 45 | hostname: broker-4-lb 46 | cap_add: 47 | - NET_ADMIN 48 | volumes: 49 | - ./nginx/broker-4.conf:/etc/nginx/nginx.conf 50 | 51 | -------------------------------------------------------------------------------- /kafka-lb/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:latest 2 | 3 | RUN \ 4 | apt-get update && \ 5 | apt-get install -y curl iproute2 iputils-ping 6 | 7 | #RUN \ 8 | # apt-get install -y blockade 9 | 10 | 11 | COPY ./broker_latency.sh /root/ 12 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-1.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-1:9094; 38 | } 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-2.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-2:9094; 38 | } 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-3.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-3:9094; 38 | } 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker-4.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | #tcp_nopush on; 25 | 26 | keepalive_timeout 65; 27 | 28 | #gzip on; 29 | 30 | include /etc/nginx/conf.d/*.conf; 31 | } 32 | 33 | stream { 34 | 35 | server { 36 | listen 9094; 37 | proxy_pass broker-4:9094; 38 | 39 | } 40 | 41 | } 42 | 43 | 44 | -------------------------------------------------------------------------------- /kafka-lb/nginx/broker_latency.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LATENCY="${1}ms" 4 | shift 5 | 6 | echo "latency: $LATENCY" 7 | 8 | tc qdisc del dev eth0 root 9 | tc qdisc add dev eth0 root handle 1: prio 10 | tc qdisc add dev eth0 parent 1:3 handle 30: netem delay ${LATENCY} 11 | 12 | 13 | declare -a HOSTS=(broker-1 broker-2 broker-3 broker-4) 14 | 15 | for host in "${HOSTS[@]}"; do 16 | 17 | ip=$(ping -c 1 $host | head -1 | awk -F'[()]' '/PING/{print $2}') 18 | 19 | echo "$host : $ip" 20 | tc filter add dev eth0 protocol ip parent 1:0 prio 3 u32 match ip dst $ip flowid 1:3 21 | 22 | done 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /kafka/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=kafka 2 | -------------------------------------------------------------------------------- /kafka/build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | jcenter() 4 | } 5 | dependencies { 6 | classpath "com.avast.gradle:gradle-docker-compose-plugin:0.12.1" 7 | } 8 | } 9 | 10 | apply plugin: 'docker-compose' 11 | 12 | /** 13 | * Make sure 'projectName' is the same as the COMPOSE_PROJECT_NAME within .env, this is to ensure the same project 14 | * name is used if you execute docker compose through gradle or from the command line. 15 | */ 16 | dockerCompose { 17 | projectName = "kdk" 18 | } 19 | -------------------------------------------------------------------------------- /kafka/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/kafka/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /kafka/jmx_prometheus/jolokia-agent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/kafka/jmx_prometheus/jolokia-agent.jar -------------------------------------------------------------------------------- /ksqlDB/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=ksqldb 2 | -------------------------------------------------------------------------------- /ksqlDB/ext/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/ksqlDB/ext/.gitkeep -------------------------------------------------------------------------------- /ksqlDB/jmx_prometheus/jmx_prometheus_javaagent.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/ksqlDB/jmx_prometheus/jmx_prometheus_javaagent.jar -------------------------------------------------------------------------------- /mongo/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=mongo 2 | -------------------------------------------------------------------------------- /mongo/db-init/00-create-user.js: -------------------------------------------------------------------------------- 1 | db.createUser({ 2 | user: "admin", 3 | pwd: "mongo", 4 | roles: [ { role: "readWrite", db: "admin" } ] 5 | }) 6 | 7 | db.createCollection("DB") 8 | -------------------------------------------------------------------------------- /mongo/restheart/users.yml: -------------------------------------------------------------------------------- 1 | ## configuration file for simpleFileAuthenticator 2 | --- 3 | users: 4 | - userid: user 5 | password: restheart 6 | roles: [user] 7 | 8 | - userid: admin 9 | password: restheart 10 | roles: [admin] 11 | 12 | -------------------------------------------------------------------------------- /mongo/secret.key: -------------------------------------------------------------------------------- 1 | wH2UDb9TINATePlnxCleLezIOv8EWI0cnI6SBap1FX4dVfwIr80I7dqNfoYLgx+d 2 | 7VXRhTlO/9p1N8+14gT1ncrplRykAX7o23TUN2eYEPpIIE4eDCpCSLYMRi1il+7/ 3 | tm0t9Q1o3mNJOQeSCBXQJuJX+NJYHyhFCMLmYAmOB8TQAUyphJZiirN+xYfaKCRZ 4 | RJxQsrdPtSG4HayHGS2xE21MrlXX5nZqZcblskjJfppDoxIFsP2saEA3FElz/bBu 5 | yGD4IXOKNEsqfpKCIyyRANgRpr71LM+JI9qRD2LuIpOweZRG/ivx5dO9ZV6v0BW0 6 | 89mdL7D9NZbHqfcGQGLP/ZmeGy8lV89C3Cwvoa4lu2f7ej4I2mov96h6ZwX1mXH1 7 | 6Pup6v5IOygc4z4ibDFUsemth0i+iLwp7/YWB6B/QaFIDiNUm3hPm4tQvYgmCU/C 8 | wpsAw+sFxq5aygC+w0Yg3k70oSyvlCF2GgqWd73O2Ek9zKum88raamOVk6Ex1BBX 9 | 402MGT25kCJW2uQoZj8+jYn1tkgGqhDTMa0ZUNa1AbWAQWcYvKKoBO3BXYNkTsRE 10 | Wi2jfwvVjwAQHqoX0trEcKsT0sGwV5Poe8F+Jh5YJVsehr283n8D2CDpKbsUDvGV 11 | hN/Yef7WQbae3hHCl7w5YN0jl2zCZ1coq/XSlymOu3j5pHskEHkoZPtOJsbj8dCs 12 | OHDXdVgIkOgUD9d2XeTXsaVbERuXhonzSK7H4AfgSA8LDPdS696NxmrIACB5j44+ 13 | Q6UBzl8sFd1UpKFKIs9fYsvmwikw/r6FWiQ/8mZKsPikjjZ6NZvGXGIipCRbEdCA 14 | +d5BQ7Eo45DQ2x9qew1+RPBt+pPVTnm25JqaXM7IsvEhPiM3q+GlPYg2sAseq3Qo 15 | qz2zQ5Z1R7oHUgSdLEn6FbGTmwDWKkgsRI7ml55KUFdp0LwKSRuc9Kf9x2BHaUX2 16 | KP69FPU7zAysyyOc6HxSQ0Pa4abRmpzvJtxG4qbGhFTOJcVg 17 | -------------------------------------------------------------------------------- /mongo/secret.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # means to generate a new secret key 5 | # 6 | 7 | openssl rand -base64 756 > secret.key 8 | 9 | -------------------------------------------------------------------------------- /monitoring/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=monitoring 2 | -------------------------------------------------------------------------------- /monitoring/akhq/application.yml: -------------------------------------------------------------------------------- 1 | akhq: 2 | connections: 3 | kafka: 4 | properties: 5 | bootstrap.servers: "broker-1:9092,broker-2:9092,broker-3:9092,broker-4:9092" 6 | security.protocol: PLAINTEXT 7 | 8 | -------------------------------------------------------------------------------- /monitoring/cmak/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11 2 | 3 | ENV ZK_HOSTS=localhost:2181 4 | ENV CMAK_VERSION=3.0.0.5 5 | 6 | RUN cd /tmp && wget https://github.com/yahoo/CMAK/archive/${CMAK_VERSION}.tar.gz && \ 7 | tar -xzvf ${CMAK_VERSION}.tar.gz 8 | 9 | RUN cd /tmp/CMAK-${CMAK_VERSION} && \ 10 | ./sbt clean dist 11 | 12 | RUN unzip -d / /tmp/CMAK-${CMAK_VERSION}/target/universal/cmak-${CMAK_VERSION}.zip 13 | 14 | RUN rm -fr /tmp/CMAK-${CMAK_VERSION} /tmp/${CMAK_VERSION}.tar.gz 15 | 16 | WORKDIR /cmak-${CMAK_VERSION} 17 | 18 | EXPOSE 9001 19 | ENTRYPOINT ["./bin/cmak","-Dconfig.file=conf/application.conf"] 20 | -------------------------------------------------------------------------------- /monitoring/grafana/dashboards.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'default' 5 | orgId: 1 6 | folder: '' 7 | folderUid: '' 8 | type: file 9 | disableDeletion: true 10 | editable: false 11 | allowUiUpdates: false 12 | updateIntervalSeconds: 5 13 | options: 14 | path: /var/lib/grafana/dashboards 15 | -------------------------------------------------------------------------------- /monitoring/grafana/druid.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: druid 5 | type: grafadruid-druid-datasource 6 | # this plugin actually does not use 'url', but by adding it it shows up nicely in the list of datasources 7 | url: http://druid-router:8888 8 | jsonData: 9 | # this is the actual propety used by this plugin for communicating with the druid router 10 | connection.url: "http://druid-router:8888" 11 | editable: false 12 | -------------------------------------------------------------------------------- /monitoring/grafana/prometheus.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: prometheus 5 | type: prometheus 6 | url: http://prometheus:9090 7 | access: proxy 8 | editable: false 9 | 10 | -------------------------------------------------------------------------------- /monitoring/kouncil/kouncil.yaml: -------------------------------------------------------------------------------- 1 | kouncil: 2 | clusters: 3 | - name: kafka 4 | brokers: 5 | - host: broker-1 6 | port: 9092 7 | jmxPort: 7071 8 | - host: broker-2 9 | port: 9092 10 | jmxPort: 7071 11 | - host: broker-3 12 | port: 9092 13 | jmxPort: 7071 14 | - host: broker-4 15 | port: 9092 16 | jmxPort: 7071 17 | -------------------------------------------------------------------------------- /monitoring/kowl/config.yaml: -------------------------------------------------------------------------------- 1 | kafka: 2 | brokers: 3 | - broker-1:9092 4 | - broker-2:9092 5 | - broker-3:9092 6 | - broker-4:9092 7 | 8 | # server: 9 | # listenPort: 8080 10 | -------------------------------------------------------------------------------- /monitoring/prometheus/targets.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "targets": [ 4 | ], 5 | "labels": { 6 | "job": "producers" 7 | } 8 | }, 9 | { 10 | "targets": [ 11 | ], 12 | "labels": { 13 | "job": "consumers" 14 | } 15 | }, 16 | { 17 | "targets": [ 18 | "stream:7071" 19 | ], 20 | "labels": { 21 | "job": "streams" 22 | } 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /monitoring/static/consumer_request_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/monitoring/static/consumer_request_metrics.png -------------------------------------------------------------------------------- /monitoring/static/producer_request_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nbuesing/kafka-local/ff2da70d0e691a00cbed68f914ee454e879ba3d6/monitoring/static/producer_request_metrics.png -------------------------------------------------------------------------------- /mysql/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=mysql 2 | -------------------------------------------------------------------------------- /mysql/cnf/local.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | local-infile 3 | secure-file-priv=/data 4 | #secure-file-priv = "" 5 | 6 | [mysql] 7 | local-infile 8 | -------------------------------------------------------------------------------- /mysql/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | volumes: 10 | mysql: {} 11 | 12 | services: 13 | mysql: 14 | image: mysql:8.0.29 15 | hostname: mysql 16 | container_name: mysql 17 | command: --default-authentication-plugin=mysql_native_password 18 | ports: 19 | - "3306:3306" 20 | healthcheck: 21 | test: exit 0 22 | interval: 15s 23 | timeout: 5s 24 | retries: 30 25 | start_period: 10s 26 | volumes: 27 | - mysql:/var/lib/mysql 28 | - ./cnf/local.cnf:/etc/mysql/conf.d/local.cnf 29 | - ./init.d:/docker-entrypoint-initdb.d 30 | - ./data:/data 31 | environment: 32 | MYSQL_ROOT_PASSWORD: mysql 33 | MYSQL_USER: user 34 | MYSQL_PASSWORD: userpw 35 | 36 | -------------------------------------------------------------------------------- /mysql/init.d/backup.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE DATABASE BACKUP; 3 | 4 | GRANT ALL PRIVILEGES ON BACKUP.* TO 'user'@'%'; 5 | 6 | USE BACKUP; 7 | 8 | DROP TABLE IF EXISTS `EXAMPLE`; 9 | 10 | /*!40101 SET @saved_cs_client = @@character_set_client */; 11 | /*!50503 SET character_set_client = utf8mb4 */; 12 | CREATE TABLE `EXAMPLE` ( 13 | `KEY` varchar(20) NOT NULL, 14 | `VALUE` varchar(250), 15 | PRIMARY KEY (`KEY`) 16 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 17 | -------------------------------------------------------------------------------- /mysql/init.d/main.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE DATABASE main; 3 | 4 | GRANT ALL PRIVILEGES ON main.* TO 'user'@'%' with grant option; 5 | 6 | GRANT SUPER ON *.* TO 'user'@'%'; 7 | 8 | USE main; 9 | 10 | -------------------------------------------------------------------------------- /mysql/mysql.cnf: -------------------------------------------------------------------------------- 1 | # For advice on how to change settings please see 2 | # http://dev.mysql.com/doc/refman/5.7/en/server-configuration-defaults.html 3 | 4 | 5 | [mysql] 6 | local-infile=1 7 | 8 | [mysqld] 9 | # 10 | # Remove leading # and set to the amount of RAM for the most important data 11 | # cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%. 12 | # innodb_buffer_pool_size = 128M 13 | # 14 | # Remove leading # to turn on a very important data integrity option: logging 15 | # changes to the binary log between backups. 16 | # log_bin 17 | # 18 | # Remove leading # to set options mainly useful for reporting servers. 19 | # The server defaults are faster for transactions and fast SELECTs. 20 | # Adjust sizes as needed, experiment to find the optimal values. 21 | # join_buffer_size = 128M 22 | # sort_buffer_size = 2M 23 | # read_rnd_buffer_size = 2M 24 | skip-host-cache 25 | skip-name-resolve 26 | #datadir=/var/lib/mysql 27 | #socket=/var/lib/mysql/mysql.sock 28 | #secure-file-priv=/var/lib/mysql-files 29 | user=mysql 30 | local-infile=1 31 | 32 | # Disabling symbolic-links is recommended to prevent assorted security risks 33 | symbolic-links=0 34 | 35 | #log-error=/var/log/mysqld.log 36 | #pid-file=/var/run/mysqld/mysqld.pid 37 | 38 | # ---------------------------------------------- 39 | # Enable the binlog for replication & CDC 40 | # ---------------------------------------------- 41 | 42 | # Enable binary replication log and set the prefix, expiration, and log format. 43 | # The prefix is arbitrary, expiration can be short for integration tests but would 44 | # be longer on a production system. Row-level info is required for ingest to work. 45 | # Server ID is required, but this will vary on production systems 46 | server-id = 111111 47 | log_bin = mysql-bin 48 | expire_logs_days = 100 49 | binlog_format = row 50 | 51 | 52 | -------------------------------------------------------------------------------- /mysql5/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=mysql5 2 | -------------------------------------------------------------------------------- /mysql5/cnf/local.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | local-infile 3 | secure-file-priv=/data 4 | #secure-file-priv = "" 5 | 6 | [mysql] 7 | local-infile 8 | -------------------------------------------------------------------------------- /mysql5/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | volumes: 10 | mysql5: {} 11 | 12 | services: 13 | mysql5: 14 | image: mysql:5.7.35 15 | hostname: mysql5 16 | container_name: mysql5 17 | ports: 18 | - "3307:3306" 19 | volumes: 20 | - ./init.d:/docker-entrypoint-initdb.d 21 | environment: 22 | MYSQL_ROOT_PASSWORD: mysql 23 | MYSQL_USER: user 24 | MYSQL_PASSWORD: userpw 25 | 26 | -------------------------------------------------------------------------------- /mysql5/init.d/main.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE DATABASE MAIN; 3 | 4 | GRANT ALL PRIVILEGES ON MAIN.* TO 'user'@'%'; 5 | 6 | USE MAIN; 7 | 8 | DROP TABLE IF EXISTS `EXAMPLE`; 9 | 10 | 11 | CREATE TABLE `EXAMPLE` ( 12 | `KEY` varchar(20) NOT NULL, 13 | `VALUE` varchar(250), 14 | PRIMARY KEY (`KEY`) 15 | ); 16 | -------------------------------------------------------------------------------- /mysql5/mysql.cnf: -------------------------------------------------------------------------------- 1 | # For advice on how to change settings please see 2 | # http://dev.mysql.com/doc/refman/5.7/en/server-configuration-defaults.html 3 | 4 | [mysqld] 5 | # 6 | # Remove leading # and set to the amount of RAM for the most important data 7 | # cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%. 8 | # innodb_buffer_pool_size = 128M 9 | # 10 | # Remove leading # to turn on a very important data integrity option: logging 11 | # changes to the binary log between backups. 12 | # log_bin 13 | # 14 | # Remove leading # to set options mainly useful for reporting servers. 15 | # The server defaults are faster for transactions and fast SELECTs. 16 | # Adjust sizes as needed, experiment to find the optimal values. 17 | # join_buffer_size = 128M 18 | # sort_buffer_size = 2M 19 | # read_rnd_buffer_size = 2M 20 | skip-host-cache 21 | skip-name-resolve 22 | #datadir=/var/lib/mysql 23 | #socket=/var/lib/mysql/mysql.sock 24 | #secure-file-priv=/var/lib/mysql-files 25 | user=mysql 26 | 27 | # Disabling symbolic-links is recommended to prevent assorted security risks 28 | symbolic-links=0 29 | 30 | #log-error=/var/log/mysqld.log 31 | #pid-file=/var/run/mysqld/mysqld.pid 32 | 33 | # ---------------------------------------------- 34 | # Enable the binlog for replication & CDC 35 | # ---------------------------------------------- 36 | 37 | # Enable binary replication log and set the prefix, expiration, and log format. 38 | # The prefix is arbitrary, expiration can be short for integration tests but would 39 | # be longer on a production system. Row-level info is required for ingest to work. 40 | # Server ID is required, but this will vary on production systems 41 | server-id = 111111 42 | log_bin = mysql-bin 43 | expire_logs_days = 100 44 | binlog_format = row 45 | 46 | 47 | -------------------------------------------------------------------------------- /network.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NETWORK=kafka-local 4 | 5 | # 6 | # creates a network unique to this project that can be shared between docker compose instances 7 | # 8 | if [ "$(docker network inspect -f '{{.Name}}' $NETWORK 2>/dev/null)" != "$NETWORK" ]; then 9 | echo "creating network $NETWORK" 10 | (docker network create $NETWORK >/dev/null) 11 | fi 12 | 13 | -------------------------------------------------------------------------------- /oracle/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=oracle 2 | -------------------------------------------------------------------------------- /oracle/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | volumes: 10 | data: {} 11 | fra: {} 12 | 13 | services: 14 | 15 | oracle: 16 | image: oracle/database:19.3.0-se2 17 | #image: oracle/database:12.2.0.1-se2 18 | hostname: oracle 19 | container_name: oracle 20 | restart: always 21 | ports: 22 | - "1521:1521" 23 | volumes: 24 | - ./init.d:/opt/oracle/scripts 25 | - data:/data 26 | - fra:/u01/app/oracle/fast_recovery_area 27 | environment: 28 | ORACLE_SID: MAIN 29 | ORACLE_PWD: oracle 30 | 31 | -------------------------------------------------------------------------------- /oracle/init.d/setup/logminer.sql: -------------------------------------------------------------------------------- 1 | 2 | SELECT SUPPLEMENTAL_LOG_DATA_MIN, SUPPLEMENTAL_LOG_DATA_ALL, FORCE_LOGGING FROM V$DATABASE; 3 | 4 | SHUTDOWN IMMEDIATE 5 | 6 | STARTUP MOUNT 7 | 8 | ALTER DATABASE ARCHIVELOG; 9 | 10 | ALTER DATABASE OPEN; 11 | 12 | ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; 13 | 14 | SELECT SUPPLEMENTAL_LOG_DATA_MIN, SUPPLEMENTAL_LOG_DATA_ALL, FORCE_LOGGING FROM V$DATABASE; 15 | 16 | ALTER SESSION SET "_ORACLE_SCRIPT"=true; 17 | 18 | CREATE ROLE LOGMNR_ROLE; 19 | GRANT CREATE SESSION TO LOGMNR_ROLE; 20 | GRANT EXECUTE_CATALOG_ROLE,SELECT ANY TRANSACTION ,SELECT ANY DICTIONARY TO LOGMNR_ROLE; 21 | 22 | CREATE USER C_USER IDENTIFIED BY C_PASS; 23 | GRANT LOGMNR_ROLE TO C_USER; 24 | ALTER USER C_USER QUOTA UNLIMITED ON USERS; 25 | 26 | ALTER DATABASE ADD SUPPLEMENTAL LOG DATA; 27 | ALTER SYSTEM ARCHIVE LOG CURRENT; 28 | 29 | ALTER SESSION SET CONTAINER = CDB$ROOT; 30 | GRANT SELECT ON SYS.USER$ TO C_USER; 31 | 32 | -------------------------------------------------------------------------------- /pinot/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=pinot 2 | -------------------------------------------------------------------------------- /postgres/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=postgres 2 | -------------------------------------------------------------------------------- /postgres/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.7' 3 | 4 | networks: 5 | default: 6 | external: true 7 | name: kafka-local 8 | 9 | volumes: 10 | postgres: {} 11 | 12 | services: 13 | postgres: 14 | image: postgres:14.3 15 | hostname: postgres 16 | container_name: postgres 17 | healthcheck: 18 | test: pg_isready -U main 19 | interval: 10s 20 | timeout: 5s 21 | retries: 5 22 | ports: 23 | - "5432:5432" 24 | command: "postgres -c wal_level=logical" 25 | volumes: 26 | - postgres:/var/lib/postgresql/data 27 | - ./data:/data 28 | environment: 29 | POSTGRES_PASSWORD: postgres 30 | POSTGRES_USER: admin 31 | POSTGRES_DB: main 32 | 33 | -------------------------------------------------------------------------------- /scripts/.zshrc: -------------------------------------------------------------------------------- 1 | # 2 | # Example aliases to consider for your local environment 3 | # 4 | 5 | alias d='docker' 6 | alias dc='docker compose' 7 | alias dps='docker ps --format "table {{.ID}}\t{{.Names}}\t{{.RunningFor}}\t{{.Status}}\t{{.Size}}\t{{.Ports}}"' 8 | 9 | alias kt='kafka-topics --bootstrap-server localhost:19092,localhost:29092,localhost:39092' 10 | 11 | alias kcc='kafka-console-consumer \ 12 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 13 | --key-deserializer=org.apache.kafka.common.serialization.BytesDeserializer \ 14 | --property print.key=true \ 15 | --property key.separator=" | " \ 16 | --from-beginning \ 17 | --topic ' 18 | 19 | alias kccv='kafka-console-consumer \ 20 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 21 | --from-beginning \ 22 | --topic' 23 | 24 | alias kacc='kafka-avro-console-consumer \ 25 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 26 | --property schema.registry.url="http://localhost:8081" \ 27 | --key-deserializer=org.apache.kafka.common.serialization.BytesDeserializer \ 28 | --property print.key=true \ 29 | --property key.separator=" | " \ 30 | --from-beginning --topic' 31 | 32 | alias kaccv='kafka-avro-console-consumer \ 33 | --bootstrap-server localhost:19092,localhost:29092,localhost:39092 \ 34 | --property schema.registry.url="http://localhost:8081" \ 35 | --from-beginning --topic' 36 | -------------------------------------------------------------------------------- /scripts/druid_download_extensions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname $0)/.. 4 | 5 | #if [ ! -f ./prometheus-emitter-0.22.0.jar ]; then 6 | # curl -s -O https://repo1.maven.org/maven2/org/apache/druid/extensions/contrib/prometheus-emitter/0.22.0/prometheus-emitter-0.22.0.jar 7 | #fi 8 | 9 | druid-protobuf-extension 10 | 11 | 12 | docker run -it --rm --entrypoint '' -v $(pwd)/druid/extensions:/opt/druid/extensions -v "druid_shared:/opt/shared" "apache/druid:0.22.0" sh -c "java -cp \"lib/*\" -Ddruid.extensions.directory=\"extensions\" -Ddruid.extensions.hadoopDependenciesDir=\"hadoop-dependencies\" org.apache.druid.cli.Main tools pull-deps --no-default-hadoop -c \"org.apache.druid.extensions.contrib:druid-protobuf-extension:0.22.0\"" 13 | 14 | 15 | -------------------------------------------------------------------------------- /scripts/druid_dump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "" 5 | echo "Usage: $0 {-mm|-h} [-i=interval} datasource" 6 | echo "" 7 | } 8 | 9 | SOURCE=h 10 | 11 | interval="" 12 | 13 | for i in "$@"; do 14 | case $i in 15 | --middlemanager|-m|-mm) 16 | SOURCE="mm" 17 | ;; 18 | --historical|-h) 19 | SOURCE="h" 20 | ;; 21 | --interval=*|-i=*) 22 | interval="${i#*=}" 23 | ;; 24 | *) 25 | datasource=$i 26 | ;; 27 | esac 28 | shift 29 | done 30 | 31 | if [ -z "${datasource}" ]; then 32 | usage 33 | exit 34 | fi 35 | 36 | 37 | echo "" 38 | echo $interval 39 | echo "" 40 | 41 | if [ "${SOURCE}" == "mm" ]; then 42 | 43 | docker exec -it druid-middlemanager sh -c "/opt/staging/scripts/export_rt.sh ${datasource}" 44 | 45 | # docker exec -it druid-middlemanager sh -c 'cd /opt/staging; FILE=$(find /opt/druid/var/druid/task -name 00000.smoosh -print -quit) && java -classpath "/opt/druid/lib/*" -Ddruid.extensions.loadList="[]" org.apache.druid.cli.Main tools dump-segment --directory $(dirname $FILE) --out ./x.txt ${COMMAND} 2>/dev/null && cat ./x.txt' 46 | 47 | else 48 | 49 | docker exec -it druid-historical sh -c "/opt/staging/scripts/export_historical.sh ${datasource} ${interval}" 50 | #docker exec -it druid-historical sh -c "/opt/staging/scripts/export_historical.sh ${datasource} 2021-12-05T13:00:00.000Z" 51 | 52 | fi 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | #docker run -it --rm --entrypoint '' -v "druid_shared:/opt/shared" "apache/druid:0.22.0" sh -c 'cd /tmp; FILE=$(find /opt/shared -name index.zip -print -quit) && unzip $FILE >/dev/null && java -classpath "/opt/druid/lib/*" -Ddruid.extensions.loadList="[]" org.apache.druid.cli.Main tools dump-segment --directory . --out ./x.txt --time-iso8601 --dump bitmaps --decompress-bitmaps 2>/dev/null && cat ./x.txt' 61 | #docker run -it --rm --entrypoint '' -v "druid_shared:/opt/shared" "apache/druid:0.22.0" sh -c "find /opt/shared -name index.zip -exec xargs -L1 echo \;" 62 | -------------------------------------------------------------------------------- /scripts/install_jmxterm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0)/.. 4 | 5 | if [ ! -f ./kafka/jmx_prometheus/jmxterm.jar ]; then 6 | 7 | echo "" 8 | echo "downloading jmxterm.jar" 9 | echo "" 10 | curl -s -L -o ./kafka/jmx_prometheus/jmxterm.jar https://github.com/jiaqi/jmxterm/releases/download/v1.0.2/jmxterm-1.0.2-uber.jar 11 | 12 | else 13 | 14 | echo "" 15 | echo "jmxterm.jar already downloaded" 16 | echo "" 17 | 18 | fi 19 | -------------------------------------------------------------------------------- /scripts/ips.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | containers=$(docker ps --format '{{.Names}}|{{.ID}}' | sort) 4 | 5 | for container in $containers; do 6 | 7 | id=$(echo $container | cut -d\| -f2) 8 | name=$(echo $container | cut -d\| -f1) 9 | 10 | ip=$(docker inspect -f '{{.Name}} {{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $id) 11 | 12 | echo $ip 13 | 14 | done 15 | 16 | 17 | -------------------------------------------------------------------------------- /scripts/stats.sh: -------------------------------------------------------------------------------- 1 | AWK=$(cat <= 1024**i || i == 0) { 43 | printf("%.2f %s\n", total/(1024**i), binaryunits[i]); 44 | break; 45 | } 46 | } 47 | } 48 | 49 | EOF 50 | ) 51 | 52 | 53 | SIZE=$(docker stats --no-stream --format "table {{.MemUsage}}" | tail -n +2 | cut -d/ -f1 | sed -E "s/([[:digit:]]+)([[:alpha:]]+)/\1 \2/" | awk "${AWK}") 54 | 55 | echo "" 56 | docker stats --no-stream 57 | echo "" 58 | echo "Total Memory Usage: $SIZE" 59 | echo "" 60 | -------------------------------------------------------------------------------- /scripts/volume_sizes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # only works on running containers, since volume size is based on executing the du command on all volumes from within the container 5 | # 6 | 7 | containers=$(docker ps --format '{{.Names}}|{{.ID}}' | sort) 8 | 9 | for container in $containers; do 10 | 11 | id=$(echo $container | cut -d\| -f2) 12 | name=$(echo $container | cut -d\| -f1) 13 | volumes=$(docker inspect -f '{{range .Mounts}}{{ if eq .Type "volume" }}{{.Destination}} {{end}}{{end}}' $id) 14 | 15 | echo $name 16 | 17 | if [ "$volumes" != "" ]; then 18 | docker exec $id du -d 0 -h $volumes 19 | fi 20 | 21 | echo "" 22 | 23 | done 24 | 25 | 26 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-local' 2 | 3 | buildscript { 4 | repositories { 5 | mavenCentral() 6 | } 7 | } 8 | 9 | include 'druid_late_publisher' 10 | 11 | project(":druid_late_publisher").projectDir = file('demo/druid_late/publisher') 12 | -------------------------------------------------------------------------------- /storage/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=buckets 2 | -------------------------------------------------------------------------------- /storage/README.md: -------------------------------------------------------------------------------- 1 | aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket foo 2 | -------------------------------------------------------------------------------- /storage/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: true 4 | name: kafka-local 5 | 6 | volumes: 7 | minio: {} 8 | 9 | services: 10 | 11 | minio: 12 | image: minio/minio:latest 13 | container_name: minio 14 | hostname: minio 15 | ports: 16 | - "9000:9000" 17 | - "9001:9001" 18 | command: server /data --console-address ":9001" 19 | healthcheck: 20 | test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] 21 | interval: 30s 22 | timeout: 20s 23 | retries: 3 24 | volumes: 25 | - minio:/data 26 | environment: 27 | - MINIO_ROOT_USER=admin 28 | - MINIO_ROOT_PASSWORD=miniominio 29 | - MINIO_SKIP_CLIENT=no 30 | 31 | # minio-create-buckets: 32 | # image: minio/mc:latest 33 | # depends_on: 34 | # - minio 35 | # entrypoint: > 36 | # /bin/sh -c " 37 | # /usr/bin/mc config host add minio http://minio:9000 admin miniominio --api s3v4; 38 | # while ! /usr/bin/mc mb minio/public ; do echo 'waiting for minio'; sleep 1; done; 39 | # /usr/bin/mc policy set download minio/public; 40 | # exit 0; 41 | # " 42 | 43 | fake-gcs-server: 44 | image: fsouza/fake-gcs-server:1.37.12 45 | container_name: fake-gcs-server 46 | hostname: fake-gcs-server 47 | ports: 48 | - "8001:8001" 49 | volumes: 50 | - ./data/gcs:/data 51 | command: -port 8001 -scheme https -external-url https://fake-gcs-server:8001 52 | -------------------------------------------------------------------------------- /storage/mc-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "10", 3 | "aliases": { 4 | "minio": { 5 | "url": "http://minio:9000", 6 | "accessKey": "admin", 7 | "secretKey": "miniominio", 8 | "api": "s3v4", 9 | "path": "auto" 10 | } 11 | } 12 | } 13 | --------------------------------------------------------------------------------